Compare commits

...

64 Commits

Author SHA1 Message Date
fatedier
066172e9c1 Merge pull request #403 from fatedier/dev
bump version to v0.13.0
2017-07-16 13:20:42 -05:00
fatedier
d5931758b6 fix user in reload command 2017-07-17 02:14:30 +08:00
fatedier
c75c3acd21 Merge pull request #402 from fatedier/doc
update doc for v0.13.0
2017-07-16 13:12:06 -05:00
fatedier
0208ecd1d9 update doc for v0.13.0 2017-07-17 02:09:51 +08:00
fatedier
23e9845e65 Merge pull request #401 from fatedier/0.13
merge 0.13
2017-07-14 23:13:42 +08:00
fatedier
2b1ba3a946 update conf 2017-07-13 12:01:46 +08:00
fatedier
ee9ddf52cd frpc: support --reload command 2017-07-13 02:30:25 +08:00
fatedier
d246400a71 frpc: add admin server for reload configure file 2017-07-13 02:20:49 +08:00
fatedier
f63a4f0cdd frps: new parameter 'proxy_bind_addr' 2017-07-05 01:40:01 +08:00
fatedier
b743b5aaed Merge pull request #390 from lukazh/patch-1
Update README.md
2017-07-05 01:27:41 +08:00
Lukaz
9d9416ab94 Update README.md
fix a typo
2017-07-04 23:05:24 +08:00
fatedier
c081df40e1 vendor: add github.com/armon/go-socks5 2017-07-01 16:09:09 +08:00
fatedier
fe32a7c4bb doc: update 2017-07-01 16:03:13 +08:00
fatedier
7bb8c10647 plugin: add socks5 plugin 2017-07-01 15:56:48 +08:00
fatedier
0752508469 vhost: a bug fix of reading request 2017-07-01 12:13:44 +08:00
fatedier
4cc1663a5f vhost: add real ip in first request of one connection
1. fix #248 host_header_rewrite bug
2. close #270, #127
2017-07-01 01:54:37 +08:00
fatedier
b55a24a27e update mutex used in frpc control 2017-06-27 23:31:02 +08:00
fatedier
aede4e54f8 close all proxies if protocol = kcp 2017-06-27 01:59:30 +08:00
fatedier
b811a620c3 vhost: fix 404 page 2017-06-26 22:24:47 +08:00
fatedier
07fe05a9d5 update version to v0.13.0 2017-06-26 20:57:10 +08:00
fatedier
171bc8dd22 new proxy type: stcp(secret tcp) 2017-06-26 03:02:33 +08:00
fatedier
9c175d4eb5 Merge pull request #380 from IanSmith123/fixbug
fix backquote
2017-06-24 13:19:43 +08:00
Iansmith's win10
9f736558e2 fix backquote 2017-06-24 12:17:09 +08:00
fatedier
8f071dd2c2 Merge pull request #375 from fangqiuming/fangqiuming-patch-1
Fix dockerfile
2017-06-21 18:50:47 +08:00
方秋鸣
bcaf51a6ad Fix dockerfile
Fix incorrect filenames
2017-06-21 14:46:24 +08:00
fatedier
ad3cf9a64a Merge pull request #372 from fatedier/dev
bump verson to v0.12.0
2017-06-19 21:36:51 +08:00
fatedier
e3fc73dbc5 update doc 2017-06-17 18:01:08 +08:00
fatedier
f884e894f2 Merge pull request #363 from fatedier/doc
update doc
2017-06-13 12:44:47 -05:00
fatedier
d57ed7d3d8 update doc 2017-06-14 01:40:20 +08:00
fatedier
a2c318d24c update kcp mode 2017-06-13 23:36:10 +08:00
fatedier
32f8745d61 Merge pull request #360 from fatedier/doc
update doc
2017-06-11 13:46:33 -05:00
fatedier
66120fe49d update doc 2017-06-12 02:41:25 +08:00
fatedier
fca7f42b37 msg: new message CloseProxy 2017-06-11 17:22:05 +08:00
fatedier
5b303f5148 vhost: return 404 not found page if domain doesn't exist 2017-06-11 16:23:00 +08:00
fatedier
2a044c9d6d http_proxy: fix error using encryption or compression 2017-06-09 02:13:24 +08:00
fatedier
70e2aee46d format 2017-06-09 01:33:57 +08:00
fatedier
6742fa2ea8 io: WithCompression resuse snappy.Reader and snappy.Writer 2017-06-08 00:57:33 +08:00
fatedier
511503d34c io.Copy use pool buffer 2017-06-06 18:48:40 +08:00
fatedier
1eaf17fd05 fix ci 2017-06-06 01:39:06 +08:00
fatedier
04f4fd0a81 proto/tcp: fix unexpected close function, fix #332 2017-06-05 23:52:24 +08:00
fatedier
3a4d769bb3 update packages 2017-06-04 20:52:42 +08:00
fatedier
84341b7fcc vendor: add kcp-go package 2017-06-04 20:07:03 +08:00
fatedier
80ba931326 support protocol kcp 2017-06-04 19:56:21 +08:00
fatedier
7ebcc7503a Merge pull request #351 from fatedier/dev
update ISSUE_TEMPLATE
2017-06-03 10:50:40 -05:00
fatedier
74cf57feb3 update ISSUE_TEMPLATE 2017-06-03 23:48:38 +08:00
fatedier
712afed0ab Merge pull request #344 from fatedier/dev
bump version to 0.11.0
2017-06-01 10:46:16 -05:00
fatedier
e29a1330ed dashboard: add proxy start and close time 2017-05-31 02:21:15 +08:00
fatedier
44971c7918 dashboard: use gzip for static files, resolve #333 2017-05-31 01:44:18 +08:00
fatedier
7bc6c72844 dashboard: fix dashboard auth error, fix #339 2017-05-31 01:07:51 +08:00
fatedier
93461e0094 Merge pull request #340 from fatedier/http_proxy
plugin: add http_proxy
2017-05-30 03:12:16 -05:00
fatedier
03d55201b2 plugin: add http_proxy 2017-05-30 16:10:21 +08:00
fatedier
e6d82f3162 Merge pull request #336 from bingtianbaihua/refactoring
add http proxy
2017-05-27 06:27:53 -05:00
ambitioner
1af6276be9 modify 2017-05-26 21:20:54 +08:00
ambitioner
d1f5ec083a add http proxy
add http proxy

add proxy code

add proxy
2017-05-26 20:57:03 +08:00
fatedier
716ec281f6 net: add WrapReadWriteCloserConn 2017-05-26 14:17:46 +08:00
fatedier
67bfae5d23 dashboard: add frps version in Overview page 2017-05-26 12:05:39 +08:00
fatedier
f0dc3ed47b metric: clear useless proxy statistics data 2017-05-26 02:00:00 +08:00
fatedier
08b0885564 Merge pull request #335 from fatedier/start
client: add start params
2017-05-24 12:50:23 -05:00
fatedier
49b503c17b client: add start params
Proxy names specified in 'start' params divided by ',' will be started.
If it is empty or not defined, all proxies will be started.
2017-05-25 01:45:38 +08:00
fatedier
150682ec63 Merge pull request #334 from fatedier/start
client: add login_fail_exit params, default is true
2017-05-24 12:16:02 -05:00
fatedier
4dc96f41c9 client: add login_fail_exit params, default is true
if login_fail_exit is false, when frpc first login to server failed, it
    will continues relogin to server every 30 seconds.
2017-05-25 01:10:58 +08:00
fatedier
6c13b6d37a net: fix HTTP_PROXY include escape characters error, fix #275 2017-05-23 02:10:36 +08:00
fatedier
1c04de380d Merge pull request #328 from fatedier/plugin
new feature plugin and unix domian socket plugin
2017-05-22 01:12:14 -05:00
fatedier
738e5dad22 new feature plugin and unix domian socket plugin 2017-05-22 00:15:18 +08:00
307 changed files with 26835 additions and 591 deletions

View File

@@ -1,4 +1,5 @@
Issue is only used for submiting bug report and documents typo. If there are same issues or answers can be found in documents, we will close it directly.
(为了节约时间,提高处理问题的效率,不按照格式填写的 issue 将会直接关闭。)
Use the commands below to provide key information from your environment:
You do NOT have to include this information if this is a FEATURE REQUEST
@@ -9,6 +10,9 @@ You do NOT have to include this information if this is a FEATURE REQUEST
**What operating system and processor architecture are you using (`go env`)?**
**Configures you used:**
**Steps to reproduce the issue:**
1.
2.

View File

@@ -2,8 +2,7 @@ sudo: false
language: go
go:
- 1.7.5
- 1.8
- 1.8.x
install:
- make

View File

@@ -1,4 +1,4 @@
FROM golang:1.6
FROM golang:1.8
COPY . /go/src/github.com/fatedier/frp
@@ -6,8 +6,8 @@ RUN cd /go/src/github.com/fatedier/frp \
&& make \
&& mv bin/frpc /frpc \
&& mv bin/frps /frps \
&& mv conf/frpc_min.ini /frpc.ini \
&& mv conf/frps_min.ini /frps.ini \
&& mv conf/frpc.ini /frpc.ini \
&& mv conf/frps.ini /frps.ini \
&& make clean
WORKDIR /

69
Godeps/Godeps.json generated
View File

@@ -6,6 +6,10 @@
"./..."
],
"Deps": [
{
"ImportPath": "github.com/armon/go-socks5",
"Rev": "e75332964ef517daa070d7c38a9466a0d687e0a5"
},
{
"ImportPath": "github.com/davecgh/go-spew/spew",
"Comment": "v1.1.0",
@@ -30,6 +34,16 @@
"Comment": "v1.1-41-g8a45e95",
"Rev": "8a45e95fc75cb77048068a62daed98cc22fdac7c"
},
{
"ImportPath": "github.com/klauspost/cpuid",
"Comment": "v1.0",
"Rev": "09cded8978dc9e80714c4d85b0322337b0a1e5e0"
},
{
"ImportPath": "github.com/klauspost/reedsolomon",
"Comment": "1.3-1-gdde6ad5",
"Rev": "dde6ad55c5e5a6379a4e82dcca32ee407346eb6d"
},
{
"ImportPath": "github.com/pkg/errors",
"Comment": "v0.8.0-5-gc605e28",
@@ -54,14 +68,67 @@
"ImportPath": "github.com/vaughan0/go-ini",
"Rev": "a98ad7ee00ec53921f08832bc06ecf7fd600e6a1"
},
{
"ImportPath": "github.com/xtaci/kcp-go",
"Comment": "v3.17",
"Rev": "df437e2b8ec365a336200f9d9da53441cf72ed47"
},
{
"ImportPath": "github.com/xtaci/smux",
"Comment": "v1.0.5-8-g2de5471",
"Rev": "2de5471dfcbc029f5fe1392b83fe784127c4943e"
},
{
"ImportPath": "golang.org/x/crypto/blowfish",
"Rev": "e1a4589e7d3ea14a3352255d04b6f1a418845e5e"
},
{
"ImportPath": "golang.org/x/crypto/cast5",
"Rev": "e1a4589e7d3ea14a3352255d04b6f1a418845e5e"
},
{
"ImportPath": "golang.org/x/crypto/pbkdf2",
"Rev": "1f22c0103821b9390939b6776727195525381532"
"Rev": "e1a4589e7d3ea14a3352255d04b6f1a418845e5e"
},
{
"ImportPath": "golang.org/x/crypto/salsa20",
"Rev": "e1a4589e7d3ea14a3352255d04b6f1a418845e5e"
},
{
"ImportPath": "golang.org/x/crypto/salsa20/salsa",
"Rev": "e1a4589e7d3ea14a3352255d04b6f1a418845e5e"
},
{
"ImportPath": "golang.org/x/crypto/tea",
"Rev": "e1a4589e7d3ea14a3352255d04b6f1a418845e5e"
},
{
"ImportPath": "golang.org/x/crypto/twofish",
"Rev": "e1a4589e7d3ea14a3352255d04b6f1a418845e5e"
},
{
"ImportPath": "golang.org/x/crypto/xtea",
"Rev": "e1a4589e7d3ea14a3352255d04b6f1a418845e5e"
},
{
"ImportPath": "golang.org/x/net/bpf",
"Rev": "e4fa1c5465ad6111f206fc92186b8c83d64adbe1"
},
{
"ImportPath": "golang.org/x/net/context",
"Rev": "e4fa1c5465ad6111f206fc92186b8c83d64adbe1"
},
{
"ImportPath": "golang.org/x/net/internal/iana",
"Rev": "e4fa1c5465ad6111f206fc92186b8c83d64adbe1"
},
{
"ImportPath": "golang.org/x/net/internal/socket",
"Rev": "e4fa1c5465ad6111f206fc92186b8c83d64adbe1"
},
{
"ImportPath": "golang.org/x/net/ipv4",
"Rev": "e4fa1c5465ad6111f206fc92186b8c83d64adbe1"
}
]
}

View File

@@ -15,7 +15,12 @@ file:
go generate ./assets/...
fmt:
go fmt ./...
go fmt ./assets/...
go fmt ./client/...
go fmt ./cmd/...
go fmt ./models/...
go fmt ./server/...
go fmt ./utils/...
frps:
go build -o bin/frps ./cmd/frps
@@ -42,7 +47,7 @@ alltest: gotest
clean:
rm -f ./bin/frpc
rm -f ./bin/frps
cd ./test && ./clean_test.sh && cd -
cd ./tests && ./clean_test.sh && cd -
save:
godep save ./...

195
README.md
View File

@@ -18,24 +18,32 @@ frp is a fast reverse proxy to help you expose a local server behind a NAT or fi
* [Access your computer in LAN by SSH](#access-your-computer-in-lan-by-ssh)
* [Visit your web service in LAN by custom domains](#visit-your-web-service-in-lan-by-custom-domains)
* [Forward DNS query request](#forward-dns-query-request)
* [Forward unix domain socket](#forward-unix-domain-socket)
* [Expose your service in security](#expose-your-service-in-security)
* [Connect website through frpc's network](#connect-website-through-frpcs-network)
* [Features](#features)
* [Configuration File](#configuration-file)
* [Dashboard](#dashboard)
* [Authentication](#authentication)
* [Encryption and Compression](#encryption-and-compression)
* [Reload configures without frps stopped](#reload-configures-without-frps-stopped)
* [Hot-Reload frpc configuration](#hot-reload-frpc-configuration)
* [Privilege Mode](#privilege-mode)
* [Port White List](#port-white-list)
* [TCP Stream Multiplexing](#tcp-stream-multiplexing)
* [Support KCP Protocol](#support-kcp-protocol)
* [Connection Pool](#connection-pool)
* [Rewriting the Host Header](#rewriting-the-host-header)
* [Get Real IP](#get-real-ip)
* [Password protecting your web service](#password-protecting-your-web-service)
* [Custom subdomain names](#custom-subdomain-names)
* [URL routing](#url-routing)
* [Connect frps by HTTP PROXY](#connect-frps-by-http-proxy)
* [Plugin](#plugin)
* [Development Plan](#development-plan)
* [Contributing](#contributing)
* [Donation](#donation)
* [AliPay](#alipay)
* [Wechat Pay](#wechat-pay)
* [Paypal](#paypal)
<!-- vim-markdown-toc -->
@@ -143,7 +151,7 @@ However, we can expose a http or https service using frp.
### Forward DNS query request
1. Modify frps.ini, configure a reverse proxy named [dns]:
1. Modify frps.ini:
```ini
# frps.ini
@@ -178,8 +186,105 @@ However, we can expose a http or https service using frp.
`dig @x.x.x.x -p 6000 www.goolge.com`
### Forward unix domain socket
Using tcp port to connect unix domain socket like docker daemon.
Configure frps same as above.
1. Start frpc with configurations:
```ini
# frpc.ini
[common]
server_addr = x.x.x.x
server_port = 7000
[unix_domain_socket]
type = tcp
remote_port = 6000
plugin = unix_domain_socket
plugin_unix_path = /var/run/docker.sock
```
2. Get docker version by curl command:
`curl http://x.x.x.x:6000/version`
### Expose your service in security
For some services, if expose them to the public network directly will be a security risk.
**stcp(secret tcp)** help you create a proxy avoiding any one can access it.
Configure frps same as above.
1. Start frpc, forward ssh port and `remote_port` is useless:
```ini
# frpc.ini
[common]
server_addr = x.x.x.x
server_port = 7000
[secret_ssh]
type = stcp
sk = abcdefg
local_ip = 127.0.0.1
local_port = 22
```
2. Start another frpc in which you want to connect this ssh server:
```ini
# frpc.ini
[common]
server_addr = x.x.x.x
server_port = 7000
[secret_ssh_vistor]
type = stcp
role = vistor
server_name = secret_ssh
sk = abcdefg
bind_addr = 127.0.0.1
bind_port = 6000
```
3. Connect to server in LAN by ssh assuming that username is test:
`ssh -oPort=6000 test@127.0.0.1`
### Connect website through frpc's network
Configure frps same as above.
1. Start frpc with configurations:
```ini
# frpc.ini
[common]
server_addr = x.x.x.x
server_port = 7000
[http_proxy]
type = tcp
remote_port = 6000
plugin = http_proxy # or socks5
```
2. Set http proxy or socks5 proxy `x.x.x.x:6000` in your browser and visit website through frpc's network.
## Features
### Configuration File
You can find features which this document not metioned from full example configuration files.
[frps full configuration file](./conf/frps_full.ini)
[frpc full configuration file](./conf/frpc_full.ini)
### Dashboard
Check frp's status and proxies's statistics information by Dashboard.
@@ -220,9 +325,20 @@ use_encryption = true
use_compression = true
```
### Reload configures without frps stopped
### Hot-Reload frpc configuration
This feature is removed since v0.10.0.
First you need to set admin port in frpc's configure file to let it provide HTTP API for more features.
```ini
# frpc.ini
[common]
admin_addr = 127.0.0.1
admin_port = 7400
```
Then run command `frpc -c ./frpc.ini --reload` and wait for about 10 seconds to let frpc create or update or delete proxies.
**Note that parameters in [common] section won't be modified except 'start' now.**
### Privilege Mode
@@ -252,6 +368,35 @@ You can disable this feature by modify frps.ini and frpc.ini:
tcp_mux = false
```
### Support KCP Protocol
frp support kcp protocol since v0.12.0.
KCP is a fast and reliable protocol that can achieve the transmission effect of a reduction of the average latency by 30% to 40% and reduction of the maximum delay by a factor of three, at the cost of 10% to 20% more bandwidth wasted than TCP.
Using kcp in frp:
1. Enable kcp protocol in frps:
```ini
# frps.ini
[common]
bind_port = 7000
# kcp needs to bind a udp port, it can be same with 'bind_port'
kcp_bind_port = 7000
```
2. Configure the protocol used in frpc to connect frps:
```ini
# frpc.ini
[common]
server_addr = x.x.x.x
# specify the 'kcp_bind_port' in frps
server_port = 7000
protocol = kcp
```
### Connection Pool
By default, frps send message to frpc for create a new connection to backward service when getting an user request.If a proxy's connection pool is enabled, there will be a specified number of connections pre-established.
@@ -289,6 +434,14 @@ host_header_rewrite = dev.yourdomain.com
If `host_header_rewrite` is specified, the Host header will be rewritten to match the hostname portion of the forwarding address.
### Get Real IP
Features for http proxy only.
You can get user's real IP from http request header `X-Forwarded-For` and `X-Real-IP`.
**Note that now you can only get these two headers in first request of each user connection.**
### Password protecting your web service
Anyone who can guess your tunnel URL can access your local web server unless you protect it with a password.
@@ -358,22 +511,46 @@ Http requests with url prefix `/news` and `/about` will be forwarded to **web02*
frpc can connect frps using HTTP PROXY if you set os environment `HTTP_PROXY` or configure `http_proxy` param in frpc.ini file.
It only works when protocol is tcp.
```ini
# frpc.ini
[common]
server_addr = x.x.x.x
server_port = 7000
http_proxy = http://user:pwd@192.168.1.128:8080
```
### Plugin
frpc only forward request to local tcp or udp port by default.
Plugin is used for providing rich features. There are built-in plugins such as **unix_domain_socket**, **http_proxy**, **socks5** and you can see [example usage](#example-usage).
Specify which plugin to use by `plugin` parameter. Configuration parameters of plugin should be started with `plugin_`. `local_ip` and `local_port` is useless for plugin.
Using plugin **http_proxy**:
```ini
# frpc.ini
[http_proxy]
type = tcp
remote_port = 6000
plugin = http_proxy
plugin_http_user = abc
plugin_http_passwd = abc
```
`plugin_http_user` and `plugin_http_passwd` are configuration parameters used in `http_proxy` plugin.
## Development Plan
* Log http request information in frps.
* Direct reverse proxy, like haproxy.
* Load balance to different service in frpc.
* Frpc can directly be a webserver for static files.
* Full control mode, dynamically modify frpc's configure with dashboard in frps.
* P2p communicate by make udp hole to penetrate NAT.
* Client Plugin (http proxy).
* kubernetes ingress support.
@@ -384,7 +561,7 @@ Interested in getting involved? We would like to help you!
* Take a look at our [issues list](https://github.com/fatedier/frp/issues) and consider sending a Pull Request to **dev branch**.
* If you want to add a new feature, please create an issue first to describe the new feature, as well as the implementation approach. Once a proposal is accepted, create an implementation of the new features and submit it as a pull request.
* Sorry for my poor english and improvement for this document is welcome even some typo fix.
* If you have some wanderful ideas, send email to fatedier@gmail.com.
* If you have some wonderful ideas, send email to fatedier@gmail.com.
**Note: We prefer you to give your advise in [issues](https://github.com/fatedier/frp/issues), so others with a same question can search it quickly and we don't need to answer them repeatly.**
@@ -398,6 +575,10 @@ frp QQ group: 606194980
![donation-alipay](/doc/pic/donate-alipay.png)
### Wechat Pay
![donation-wechatpay](/doc/pic/donate-wechatpay.png)
### Paypal
Donate money by [paypal](https://www.paypal.me/fatedier) to my account **fatedier@gmail.com**.

View File

@@ -16,24 +16,32 @@ frp 是一个可用于内网穿透的高性能的反向代理应用,支持 tcp
* [通过 ssh 访问公司内网机器](#通过-ssh-访问公司内网机器)
* [通过自定义域名访问部署于内网的 web 服务](#通过自定义域名访问部署于内网的-web-服务)
* [转发 DNS 查询请求](#转发-dns-查询请求)
* [转发 Unix域套接字](#转发-unix域套接字)
* [安全地暴露内网服务](#安全地暴露内网服务)
* [通过 frpc 所在机器访问外网](#通过-frpc-所在机器访问外网)
* [功能说明](#功能说明)
* [配置文件](#配置文件)
* [Dashboard](#dashboard)
* [身份验证](#身份验证)
* [加密与压缩](#加密与压缩)
* [服务器端热加载配置文件](#服务器端热加载配置文件)
* [客户端热加载配置文件](#客户端热加载配置文件)
* [特权模式](#特权模式)
* [端口白名单](#端口白名单)
* [TCP 多路复用](#tcp-多路复用)
* [底层通信可选 kcp 协议](#底层通信可选-kcp-协议)
* [连接池](#连接池)
* [修改 Host Header](#修改-host-header)
* [获取用户真实 IP](#获取用户真实-ip)
* [通过密码保护你的 web 服务](#通过密码保护你的-web-服务)
* [自定义二级域名](#自定义二级域名)
* [URL 路由](#url-路由)
* [通过代理连接 frps](#通过代理连接-frps)
* [插件](#插件)
* [开发计划](#开发计划)
* [为 frp 做贡献](#为-frp-做贡献)
* [捐助](#捐助)
* [支付宝扫码捐赠](#支付宝扫码捐赠)
* [微信支付捐赠](#微信支付捐赠)
* [Paypal 捐赠](#paypal-捐赠)
<!-- vim-markdown-toc -->
@@ -179,8 +187,113 @@ DNS 查询请求通常使用 UDP 协议frp 支持对内网 UDP 服务的穿
`dig @x.x.x.x -p 6000 www.goolge.com`
### 转发 Unix域套接字
通过 tcp 端口访问内网的 unix域套接字(和 docker daemon 通信)。
frps 的部署步骤同上。
1. 启动 frpc启用 unix_domain_socket 插件,配置如下:
```ini
# frpc.ini
[common]
server_addr = x.x.x.x
server_port = 7000
[unix_domain_socket]
type = tcp
remote_port = 6000
plugin = unix_domain_socket
plugin_unix_path = /var/run/docker.sock
```
2. 通过 curl 命令查看 docker 版本信息
`curl http://x.x.x.x:6000/version`
### 安全地暴露内网服务
对于某些服务来说如果直接暴露于公网上将会存在安全隐患。
使用 **stcp(secret tcp)** 类型的代理可以避免让任何人都能访问到要穿透的服务,但是访问者也需要运行另外一个 frpc。
以下示例将会创建一个只有自己能访问到的 ssh 服务代理。
frps 的部署步骤同上。
1. 启动 frpc转发内网的 ssh 服务,配置如下,不需要指定远程端口:
```ini
# frpc.ini
[common]
server_addr = x.x.x.x
server_port = 7000
[secret_ssh]
type = stcp
# 只有 sk 一致的用户才能访问到此服务
sk = abcdefg
local_ip = 127.0.0.1
local_port = 22
```
2. 在要访问这个服务的机器上启动另外一个 frpc配置如下
```ini
# frpc.ini
[common]
server_addr = x.x.x.x
server_port = 7000
[secret_ssh_vistor]
type = stcp
# stcp 的访问者
role = vistor
# 要访问的 stcp 代理的名字
server_name = secret_ssh
sk = abcdefg
# 绑定本地端口用于访问 ssh 服务
bind_addr = 127.0.0.1
bind_port = 6000
```
3. 通过 ssh 访问内网机器,假设用户名为 test
`ssh -oPort=6000 test@127.0.0.1`
### 通过 frpc 所在机器访问外网
frpc 内置了 http proxy 和 socks5 插件,可以使其他机器通过 frpc 的网络访问互联网。
frps 的部署步骤同上。
1. 启动 frpc启用 http_proxy 或 socks5 插件(plugin 换为 socks5 即可) 配置如下:
```ini
# frpc.ini
[common]
server_addr = x.x.x.x
server_port = 7000
[http_proxy]
type = tcp
remote_port = 6000
plugin = http_proxy
```
2. 浏览器设置 http 或 socks5 代理地址为 `x.x.x.x:6000`,通过 frpc 机器的网络访问互联网。
## 功能说明
### 配置文件
由于 frp 目前支持的功能和配置项较多,未在文档中列出的功能可以从完整的示例配置文件中发现。
[frps 完整配置文件](./conf/frps_full.ini)
[frpc 完整配置文件](./conf/frpc_full.ini)
### Dashboard
通过浏览器查看 frp 的状态以及代理统计信息展示。
@@ -225,9 +338,26 @@ use_compression = true
如果传输的报文长度较长,通过设置 `use_compression = true` 对传输内容进行压缩,可以有效减小 frpc 与 frps 之间的网络流量,加快流量转发速度,但是会额外消耗一些 cpu 资源。
### 服务器端热加载配置文件
### 客户端热加载配置文件
由于从 v0.10.0 版本开始,所有 proxy 都在客户端配置,这个功能暂时移除
当修改了 frpc 中的代理配置,可以通过 `frpc --reload` 命令来动态加载配置文件,通常会在 10 秒内完成代理的更新
启用此功能需要在 frpc 中启用 admin 端口,用于提供 API 服务。配置如下:
```ini
# frpc.ini
[common]
admin_addr = 127.0.0.1
admin_port = 7400
```
之后执行重启命令:
`frpc -c ./frpc.ini --reload`
等待一段时间后客户端会根据新的配置文件创建、更新、删除代理。
**需要注意的是,[common] 中的参数除了 start 外目前无法被修改。**
### 特权模式
@@ -257,6 +387,35 @@ privilege_allow_ports 可以配置允许使用的某个指定端口或者是一
tcp_mux = false
```
### 底层通信可选 kcp 协议
从 v0.12.0 版本开始,底层通信协议支持选择 kcp 协议,在弱网环境下传输效率提升明显,但是会有一些额外的流量消耗。
开启 kcp 协议支持:
1. 在 frps.ini 中启用 kcp 协议支持,指定一个 udp 端口用于接收客户端请求:
```ini
# frps.ini
[common]
bind_port = 7000
# kcp 绑定的是 udp 端口,可以和 bind_port 一样
kcp_bind_port = 7000
```
2. 在 frpc.ini 指定需要使用的协议类型,目前只支持 tcp 和 kcp。其他代理配置不需要变更
```ini
# frpc.ini
[common]
server_addr = x.x.x.x
# server_port 指定为 frps 的 kcp_bind_port
server_port = 7000
protocol = kcp
```
3. 像之前一样使用 frp需要注意开放相关机器上的 udp 的端口的访问权限。
### 连接池
默认情况下当用户请求建立连接后frps 才会请求 frpc 主动与后端服务建立一个连接。当为指定的代理启用连接池后frp 会预先和后端服务建立起指定数量的连接,每次接收到用户请求后,会从连接池中取出一个连接和用户连接关联起来,避免了等待与后端服务建立连接以及 frpc 和 frps 之间传递控制信息的时间。
@@ -294,6 +453,12 @@ host_header_rewrite = dev.yourdomain.com
原来 http 请求中的 host 字段 `test.yourdomain.com` 转发到后端服务时会被替换为 `dev.yourdomain.com`。
### 获取用户真实 IP
目前只有 **http** 类型的代理支持这一功能,可以通过用户请求的 header 中的 `X-Forwarded-For` 和 `X-Real-IP` 来获取用户真实 IP。
**需要注意的是,目前只在每一个用户连接的第一个 HTTP 请求中添加了这两个 header。**
### 通过密码保护你的 web 服务
由于所有客户端共用一个 frps 的 http 服务端口,任何知道你的域名和 url 的人都能访问到你部署在内网的 web 服务,但是在某些场景下需要确保只有限定的用户才能访问。
@@ -373,13 +538,38 @@ locations = /news,/about
可以通过设置 `HTTP_PROXY` 系统环境变量或者通过在 frpc 的配置文件中设置 `http_proxy` 参数来使用此功能。
仅在 `protocol = tcp` 时生效。
```ini
# frpc.ini
[common]
server_addr = x.x.x.x
server_port = 7000
http_proxy = http://user:pwd@192.168.1.128:8080
```
### 插件
默认情况下frpc 只会转发请求到本地 tcp 或 udp 端口。
插件模式是为了在客户端提供更加丰富的功能,目前内置的插件有 **unix_domain_socket**、**http_proxy**、**socks5**。具体使用方式请查看[使用示例](#使用示例)。
通过 `plugin` 指定需要使用的插件,插件的配置参数都以 `plugin_` 开头。使用插件后 `local_ip` 和 `local_port` 不再需要配置。
使用 **http_proxy** 插件的示例:
```ini
# frpc.ini
[http_proxy]
type = tcp
remote_port = 6000
plugin = http_proxy
plugin_http_user = abc
plugin_http_passwd = abc
```
`plugin_http_user` 和 `plugin_http_passwd` 即为 `http_proxy` 插件可选的配置参数。
## 开发计划
计划在后续版本中加入的功能与优化,排名不分先后,如果有其他功能建议欢迎在 [issues](https://github.com/fatedier/frp/issues) 中反馈。
@@ -388,9 +578,7 @@ http_proxy = http://user:pwd@192.168.1.128:8080
* frps 支持直接反向代理,类似 haproxy。
* frpc 支持负载均衡到后端不同服务。
* frpc 支持直接作为 webserver 访问指定静态页面。
* frpc 完全控制模式,通过 dashboard 对 frpc 进行在线操作。
* 支持 udp 打洞的方式,提供两边内网机器直接通信,流量不经过服务器转发。
* 支持 pluginfrpc 获取到的连接可以交给指定 plugin 处理,例如 http 代理,简单的 web server。
* 集成对 k8s 等平台的支持。
## 为 frp 做贡献
@@ -416,6 +604,10 @@ frp 交流群606194980 (QQ 群号)
![donate-alipay](/doc/pic/donate-alipay.png)
### 微信支付捐赠
![donate-wechatpay](/doc/pic/donate-wechatpay.png)
### Paypal 捐赠
海外用户推荐通过 [Paypal](https://www.paypal.me/fatedier) 向我的账户 **fatedier@gmail.com** 进行捐赠。

View File

@@ -1 +1 @@
<!DOCTYPE html> <html lang=en> <head> <meta charset=utf-8> <title>frps dashboard</title> <link rel="shortcut icon" href="favicon.ico"></head> <body> <div id=app></div> <script type="text/javascript" src="manifest.js?b52826060da73c6b5a10"></script><script type="text/javascript" src="vendor.js?66dfcf2d1c500e900413"></script><script type="text/javascript" src="index.js?ceb589f1be7a87112dbd"></script></body> </html>
<!DOCTYPE html> <html lang=en> <head> <meta charset=utf-8> <title>frps dashboard</title> <link rel="shortcut icon" href="favicon.ico"></head> <body> <div id=app></div> <script type="text/javascript" src="manifest.js?5217927b66cc446ebfd3"></script><script type="text/javascript" src="vendor.js?66dfcf2d1c500e900413"></script><script type="text/javascript" src="index.js?bf962cded96400bef9a0"></script></body> </html>

File diff suppressed because one or more lines are too long

View File

@@ -1 +1 @@
!function(e){function r(n){if(t[n])return t[n].exports;var o=t[n]={i:n,l:!1,exports:{}};return e[n].call(o.exports,o,o.exports,r),o.l=!0,o.exports}var n=window.webpackJsonp;window.webpackJsonp=function(t,c,u){for(var i,a,f,l=0,s=[];l<t.length;l++)a=t[l],o[a]&&s.push(o[a][0]),o[a]=0;for(i in c)Object.prototype.hasOwnProperty.call(c,i)&&(e[i]=c[i]);for(n&&n(t,c,u);s.length;)s.shift()();if(u)for(l=0;l<u.length;l++)f=r(r.s=u[l]);return f};var t={},o={2:0};r.e=function(e){function n(){u.onerror=u.onload=null,clearTimeout(i);var r=o[e];0!==r&&(r&&r[1](new Error("Loading chunk "+e+" failed.")),o[e]=void 0)}if(0===o[e])return Promise.resolve();if(o[e])return o[e][2];var t=new Promise(function(r,n){o[e]=[r,n]});o[e][2]=t;var c=document.getElementsByTagName("head")[0],u=document.createElement("script");u.type="text/javascript",u.charset="utf-8",u.async=!0,u.timeout=12e4,r.nc&&u.setAttribute("nonce",r.nc),u.src=r.p+""+e+".js?"+{0:"ceb589f1be7a87112dbd",1:"66dfcf2d1c500e900413"}[e];var i=setTimeout(n,12e4);return u.onerror=u.onload=n,c.appendChild(u),t},r.m=e,r.c=t,r.i=function(e){return e},r.d=function(e,n,t){r.o(e,n)||Object.defineProperty(e,n,{configurable:!1,enumerable:!0,get:t})},r.n=function(e){var n=e&&e.__esModule?function(){return e.default}:function(){return e};return r.d(n,"a",n),n},r.o=function(e,r){return Object.prototype.hasOwnProperty.call(e,r)},r.p="",r.oe=function(e){throw console.error(e),e}}([]);
!function(e){function r(n){if(t[n])return t[n].exports;var o=t[n]={i:n,l:!1,exports:{}};return e[n].call(o.exports,o,o.exports,r),o.l=!0,o.exports}var n=window.webpackJsonp;window.webpackJsonp=function(t,c,u){for(var i,a,f,l=0,s=[];l<t.length;l++)a=t[l],o[a]&&s.push(o[a][0]),o[a]=0;for(i in c)Object.prototype.hasOwnProperty.call(c,i)&&(e[i]=c[i]);for(n&&n(t,c,u);s.length;)s.shift()();if(u)for(l=0;l<u.length;l++)f=r(r.s=u[l]);return f};var t={},o={2:0};r.e=function(e){function n(){u.onerror=u.onload=null,clearTimeout(i);var r=o[e];0!==r&&(r&&r[1](new Error("Loading chunk "+e+" failed.")),o[e]=void 0)}if(0===o[e])return Promise.resolve();if(o[e])return o[e][2];var t=new Promise(function(r,n){o[e]=[r,n]});o[e][2]=t;var c=document.getElementsByTagName("head")[0],u=document.createElement("script");u.type="text/javascript",u.charset="utf-8",u.async=!0,u.timeout=12e4,r.nc&&u.setAttribute("nonce",r.nc),u.src=r.p+""+e+".js?"+{0:"bf962cded96400bef9a0",1:"66dfcf2d1c500e900413"}[e];var i=setTimeout(n,12e4);return u.onerror=u.onload=n,c.appendChild(u),t},r.m=e,r.c=t,r.i=function(e){return e},r.d=function(e,n,t){r.o(e,n)||Object.defineProperty(e,n,{configurable:!1,enumerable:!0,get:t})},r.n=function(e){var n=e&&e.__esModule?function(){return e.default}:function(){return e};return r.d(n,"a",n),n},r.o=function(e,r){return Object.prototype.hasOwnProperty.call(e,r)},r.p="",r.oe=function(e){throw console.error(e),e}}([]);

File diff suppressed because one or more lines are too long

60
client/admin.go Normal file
View File

@@ -0,0 +1,60 @@
// Copyright 2017 fatedier, fatedier@gmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package client
import (
"fmt"
"net"
"net/http"
"time"
"github.com/fatedier/frp/models/config"
frpNet "github.com/fatedier/frp/utils/net"
"github.com/julienschmidt/httprouter"
)
var (
httpServerReadTimeout = 10 * time.Second
httpServerWriteTimeout = 10 * time.Second
)
func (svr *Service) RunAdminServer(addr string, port int64) (err error) {
// url router
router := httprouter.New()
user, passwd := config.ClientCommonCfg.AdminUser, config.ClientCommonCfg.AdminPwd
// api, see dashboard_api.go
router.GET("/api/reload", frpNet.HttprouterBasicAuth(svr.apiReload, user, passwd))
address := fmt.Sprintf("%s:%d", addr, port)
server := &http.Server{
Addr: address,
Handler: router,
ReadTimeout: httpServerReadTimeout,
WriteTimeout: httpServerWriteTimeout,
}
if address == "" {
address = ":http"
}
ln, err := net.Listen("tcp", address)
if err != nil {
return err
}
go server.Serve(ln)
return
}

78
client/admin_api.go Normal file
View File

@@ -0,0 +1,78 @@
// Copyright 2017 fatedier, fatedier@gmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package client
import (
"encoding/json"
"net/http"
"github.com/julienschmidt/httprouter"
ini "github.com/vaughan0/go-ini"
"github.com/fatedier/frp/models/config"
"github.com/fatedier/frp/utils/log"
)
type GeneralResponse struct {
Code int64 `json:"code"`
Msg string `json:"msg"`
}
// api/reload
type ReloadResp struct {
GeneralResponse
}
func (svr *Service) apiReload(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
var (
buf []byte
res ReloadResp
)
defer func() {
log.Info("Http response [/api/reload]: code [%d]", res.Code)
buf, _ = json.Marshal(&res)
w.Write(buf)
}()
log.Info("Http request: [/api/reload]")
conf, err := ini.LoadFile(config.ClientCommonCfg.ConfigFile)
if err != nil {
res.Code = 1
res.Msg = err.Error()
log.Error("reload frpc config file error: %v", err)
return
}
newCommonCfg, err := config.LoadClientCommonConf(conf)
if err != nil {
res.Code = 2
res.Msg = err.Error()
log.Error("reload frpc common section error: %v", err)
return
}
pxyCfgs, vistorCfgs, err := config.LoadProxyConfFromFile(config.ClientCommonCfg.User, conf, newCommonCfg.Start)
if err != nil {
res.Code = 3
res.Msg = err.Error()
log.Error("reload frpc proxy config error: %v", err)
return
}
svr.ctl.reloadConf(pxyCfgs, vistorCfgs)
log.Info("success reload conf")
return
}

View File

@@ -24,8 +24,9 @@ import (
"github.com/fatedier/frp/models/config"
"github.com/fatedier/frp/models/msg"
"github.com/fatedier/frp/utils/crypto"
"github.com/fatedier/frp/utils/errors"
"github.com/fatedier/frp/utils/log"
"github.com/fatedier/frp/utils/net"
frpNet "github.com/fatedier/frp/utils/net"
"github.com/fatedier/frp/utils/util"
"github.com/fatedier/frp/utils/version"
"github.com/xtaci/smux"
@@ -48,8 +49,14 @@ type Control struct {
// proxies
proxies map[string]Proxy
// vistor configures
vistorCfgs map[string]config.ProxyConf
// vistors
vistors map[string]Vistor
// control connection
conn net.Conn
conn frpNet.Conn
// tcp stream multiplexing, if enabled
session *smux.Session
@@ -63,8 +70,8 @@ type Control struct {
// run id got from server
runId string
// connection or other error happens , control will try to reconnect to server
closed int32
// if we call close() in control, do not reconnect to server
exit bool
// goroutines can block by reading from this channel, it will be closed only in reader() when control connection is closed
closedCh chan int
@@ -77,7 +84,7 @@ type Control struct {
log.Logger
}
func NewControl(svr *Service, pxyCfgs map[string]config.ProxyConf) *Control {
func NewControl(svr *Service, pxyCfgs map[string]config.ProxyConf, vistorCfgs map[string]config.ProxyConf) *Control {
loginMsg := &msg.Login{
Arch: runtime.GOARCH,
Os: runtime.GOOS,
@@ -86,14 +93,16 @@ func NewControl(svr *Service, pxyCfgs map[string]config.ProxyConf) *Control {
Version: version.Full(),
}
return &Control{
svr: svr,
loginMsg: loginMsg,
pxyCfgs: pxyCfgs,
proxies: make(map[string]Proxy),
sendCh: make(chan msg.Message, 10),
readCh: make(chan msg.Message, 10),
closedCh: make(chan int),
Logger: log.NewPrefixLogger(""),
svr: svr,
loginMsg: loginMsg,
pxyCfgs: pxyCfgs,
vistorCfgs: vistorCfgs,
proxies: make(map[string]Proxy),
vistors: make(map[string]Vistor),
sendCh: make(chan msg.Message, 10),
readCh: make(chan msg.Message, 10),
closedCh: make(chan int),
Logger: log.NewPrefixLogger(""),
}
}
@@ -105,10 +114,22 @@ func NewControl(svr *Service, pxyCfgs map[string]config.ProxyConf) *Control {
// 6. In controler(): ini readCh, sendCh, closedCh
// 7. In controler(): start new reader(), writer(), manager()
// controler() will keep running
func (ctl *Control) Run() error {
err := ctl.login()
if err != nil {
return err
func (ctl *Control) Run() (err error) {
for {
err = ctl.login()
if err != nil {
ctl.Warn("login to server failed: %v", err)
// if login_fail_exit is true, just exit this program
// otherwise sleep a while and continues relogin to server
if config.ClientCommonCfg.LoginFailExit {
return
} else {
time.Sleep(30 * time.Second)
}
} else {
break
}
}
go ctl.controler()
@@ -116,6 +137,18 @@ func (ctl *Control) Run() error {
go ctl.writer()
go ctl.reader()
// start all local vistors
for _, cfg := range ctl.vistorCfgs {
vistor := NewVistor(ctl, cfg)
err = vistor.Run()
if err != nil {
vistor.Warn("start error: %v", err)
continue
}
ctl.vistors[cfg.GetName()] = vistor
vistor.Info("start vistor success")
}
// send NewProxy message for all configured proxies
for _, cfg := range ctl.pxyCfgs {
var newProxyMsg msg.NewProxy
@@ -126,29 +159,13 @@ func (ctl *Control) Run() error {
}
func (ctl *Control) NewWorkConn() {
var (
workConn net.Conn
err error
)
if config.ClientCommonCfg.TcpMux {
stream, err := ctl.session.OpenStream()
if err != nil {
ctl.Warn("start new work connection error: %v", err)
return
}
workConn = net.WrapConn(stream)
} else {
workConn, err = net.ConnectTcpServerByHttpProxy(config.ClientCommonCfg.HttpProxy,
fmt.Sprintf("%s:%d", config.ClientCommonCfg.ServerAddr, config.ClientCommonCfg.ServerPort))
if err != nil {
ctl.Warn("start new work connection error: %v", err)
return
}
workConn, err := ctl.connectServer()
if err != nil {
return
}
m := &msg.NewWorkConn{
RunId: ctl.runId,
RunId: ctl.getRunId(),
}
if err = msg.WriteMsg(workConn, m); err != nil {
ctl.Warn("work connection write to server error: %v", err)
@@ -165,14 +182,29 @@ func (ctl *Control) NewWorkConn() {
workConn.AddLogPrefix(startMsg.ProxyName)
// dispatch this work connection to related proxy
if pxy, ok := ctl.proxies[startMsg.ProxyName]; ok {
workConn.Info("start a new work connection, localAddr: %s remoteAddr: %s", workConn.LocalAddr().String(), workConn.RemoteAddr().String())
pxy, ok := ctl.getProxy(startMsg.ProxyName)
if ok {
workConn.Debug("start a new work connection, localAddr: %s remoteAddr: %s", workConn.LocalAddr().String(), workConn.RemoteAddr().String())
go pxy.InWorkConn(workConn)
} else {
workConn.Close()
}
}
func (ctl *Control) Close() error {
ctl.mu.Lock()
ctl.exit = true
err := errors.PanicToError(func() {
for name, _ := range ctl.proxies {
ctl.sendCh <- &msg.CloseProxy{
ProxyName: name,
}
}
})
ctl.mu.Unlock()
return err
}
func (ctl *Control) init() {
ctl.sendCh = make(chan msg.Message, 10)
ctl.readCh = make(chan msg.Message, 10)
@@ -188,7 +220,7 @@ func (ctl *Control) login() (err error) {
ctl.session.Close()
}
conn, err := net.ConnectTcpServerByHttpProxy(config.ClientCommonCfg.HttpProxy,
conn, err := frpNet.ConnectServerByHttpProxy(config.ClientCommonCfg.HttpProxy, config.ClientCommonCfg.Protocol,
fmt.Sprintf("%s:%d", config.ClientCommonCfg.ServerAddr, config.ClientCommonCfg.ServerPort))
if err != nil {
return err
@@ -210,14 +242,14 @@ func (ctl *Control) login() (err error) {
session.Close()
return errRet
}
conn = net.WrapConn(stream)
conn = frpNet.WrapConn(stream)
ctl.session = session
}
now := time.Now().Unix()
ctl.loginMsg.PrivilegeKey = util.GetAuthKey(config.ClientCommonCfg.PrivilegeToken, now)
ctl.loginMsg.Timestamp = now
ctl.loginMsg.RunId = ctl.runId
ctl.loginMsg.RunId = ctl.getRunId()
if err = msg.WriteMsg(conn, ctl.loginMsg); err != nil {
return err
@@ -238,7 +270,7 @@ func (ctl *Control) login() (err error) {
ctl.conn = conn
// update runId got from server
ctl.runId = loginRespMsg.RunId
ctl.setRunId(loginRespMsg.RunId)
ctl.ClearLogPrefix()
ctl.AddLogPrefix(loginRespMsg.RunId)
ctl.Info("login to server success, get run id [%s]", loginRespMsg.RunId)
@@ -250,6 +282,27 @@ func (ctl *Control) login() (err error) {
return nil
}
func (ctl *Control) connectServer() (conn frpNet.Conn, err error) {
if config.ClientCommonCfg.TcpMux {
stream, errRet := ctl.session.OpenStream()
if errRet != nil {
err = errRet
ctl.Warn("start new connection to server error: %v", err)
return
}
conn = frpNet.WrapConn(stream)
} else {
conn, err = frpNet.ConnectServerByHttpProxy(config.ClientCommonCfg.HttpProxy, config.ClientCommonCfg.Protocol,
fmt.Sprintf("%s:%d", config.ClientCommonCfg.ServerAddr, config.ClientCommonCfg.ServerPort))
if err != nil {
ctl.Warn("start new connection to server error: %v", err)
return
}
}
return
}
func (ctl *Control) reader() {
defer func() {
if err := recover(); err != nil {
@@ -294,6 +347,7 @@ func (ctl *Control) writer() {
}
}
// manager handles all channel events and do corresponding process
func (ctl *Control) manager() {
defer func() {
if err := recover(); err != nil {
@@ -334,22 +388,26 @@ func (ctl *Control) manager() {
ctl.Warn("[%s] start error: %s", m.ProxyName, m.Error)
continue
}
cfg, ok := ctl.pxyCfgs[m.ProxyName]
cfg, ok := ctl.getProxyConf(m.ProxyName)
if !ok {
// it will never go to this branch now
ctl.Warn("[%s] no proxy conf found", m.ProxyName)
continue
}
oldPxy, ok := ctl.proxies[m.ProxyName]
oldPxy, ok := ctl.getProxy(m.ProxyName)
if ok {
oldPxy.Close()
}
pxy := NewProxy(ctl, cfg)
if err := pxy.Run(); err != nil {
ctl.Warn("[%s] proxy start running error: %v", m.ProxyName, err)
ctl.sendCh <- &msg.CloseProxy{
ProxyName: m.ProxyName,
}
continue
}
ctl.proxies[m.ProxyName] = pxy
ctl.addProxy(m.ProxyName, pxy)
ctl.Info("[%s] start proxy success", m.ProxyName)
case *msg.Pong:
ctl.lastPong = time.Now()
@@ -359,26 +417,43 @@ func (ctl *Control) manager() {
}
}
// control keep watching closedCh, start a new connection if previous control connection is closed
// controler keep watching closedCh, start a new connection if previous control connection is closed.
// If controler is notified by closedCh, reader and writer and manager will exit, then recall these functions.
func (ctl *Control) controler() {
var err error
maxDelayTime := 30 * time.Second
delayTime := time.Second
checkInterval := 30 * time.Second
checkInterval := 10 * time.Second
checkProxyTicker := time.NewTicker(checkInterval)
for {
select {
case <-checkProxyTicker.C:
// Every 30 seconds, check which proxy registered failed and reregister it to server.
// Every 10 seconds, check which proxy registered failed and reregister it to server.
ctl.mu.RLock()
for _, cfg := range ctl.pxyCfgs {
if _, exist := ctl.proxies[cfg.GetName()]; !exist {
ctl.Info("try to reregister proxy [%s]", cfg.GetName())
ctl.Info("try to register proxy [%s]", cfg.GetName())
var newProxyMsg msg.NewProxy
cfg.UnMarshalToMsg(&newProxyMsg)
ctl.sendCh <- &newProxyMsg
}
}
for _, cfg := range ctl.vistorCfgs {
if _, exist := ctl.vistors[cfg.GetName()]; !exist {
ctl.Info("try to start vistor [%s]", cfg.GetName())
vistor := NewVistor(ctl, cfg)
err = vistor.Run()
if err != nil {
vistor.Warn("start error: %v", err)
continue
}
ctl.vistors[cfg.GetName()] = vistor
vistor.Info("start vistor success")
}
}
ctl.mu.RUnlock()
case _, ok := <-ctl.closedCh:
// we won't get any variable from this channel
if !ok {
@@ -389,6 +464,14 @@ func (ctl *Control) controler() {
for _, pxy := range ctl.proxies {
pxy.Close()
}
// if ctl.exit is true, just exit
ctl.mu.RLock()
exit := ctl.exit
ctl.mu.RUnlock()
if exit {
return
}
time.Sleep(time.Second)
// loop util reconnect to server success
@@ -418,11 +501,13 @@ func (ctl *Control) controler() {
go ctl.reader()
// send NewProxy message for all configured proxies
ctl.mu.RLock()
for _, cfg := range ctl.pxyCfgs {
var newProxyMsg msg.NewProxy
cfg.UnMarshalToMsg(&newProxyMsg)
ctl.sendCh <- &newProxyMsg
}
ctl.mu.RUnlock()
checkProxyTicker.Stop()
checkProxyTicker = time.NewTicker(checkInterval)
@@ -430,3 +515,107 @@ func (ctl *Control) controler() {
}
}
}
func (ctl *Control) setRunId(runId string) {
ctl.mu.Lock()
defer ctl.mu.Unlock()
ctl.runId = runId
}
func (ctl *Control) getRunId() string {
ctl.mu.RLock()
defer ctl.mu.RUnlock()
return ctl.runId
}
func (ctl *Control) getProxy(name string) (pxy Proxy, ok bool) {
ctl.mu.RLock()
defer ctl.mu.RUnlock()
pxy, ok = ctl.proxies[name]
return
}
func (ctl *Control) addProxy(name string, pxy Proxy) {
ctl.mu.Lock()
defer ctl.mu.Unlock()
ctl.proxies[name] = pxy
}
func (ctl *Control) getProxyConf(name string) (conf config.ProxyConf, ok bool) {
ctl.mu.RLock()
defer ctl.mu.RUnlock()
conf, ok = ctl.pxyCfgs[name]
return
}
func (ctl *Control) reloadConf(pxyCfgs map[string]config.ProxyConf, vistorCfgs map[string]config.ProxyConf) {
ctl.mu.Lock()
defer ctl.mu.Unlock()
removedPxyNames := make([]string, 0)
for name, oldCfg := range ctl.pxyCfgs {
del := false
cfg, ok := pxyCfgs[name]
if !ok {
del = true
} else {
if !oldCfg.Compare(cfg) {
del = true
}
}
if del {
removedPxyNames = append(removedPxyNames, name)
delete(ctl.pxyCfgs, name)
if pxy, ok := ctl.proxies[name]; ok {
pxy.Close()
}
delete(ctl.proxies, name)
ctl.sendCh <- &msg.CloseProxy{
ProxyName: name,
}
}
}
ctl.Info("proxy removed: %v", removedPxyNames)
addedPxyNames := make([]string, 0)
for name, cfg := range pxyCfgs {
if _, ok := ctl.pxyCfgs[name]; !ok {
ctl.pxyCfgs[name] = cfg
addedPxyNames = append(addedPxyNames, name)
}
}
ctl.Info("proxy added: %v", addedPxyNames)
removedVistorName := make([]string, 0)
for name, oldVistorCfg := range ctl.vistorCfgs {
del := false
cfg, ok := vistorCfgs[name]
if !ok {
del = true
} else {
if !oldVistorCfg.Compare(cfg) {
del = true
}
}
if del {
removedVistorName = append(removedVistorName, name)
delete(ctl.vistorCfgs, name)
if vistor, ok := ctl.vistors[name]; ok {
vistor.Close()
}
delete(ctl.vistors, name)
}
}
ctl.Info("vistor removed: %v", removedVistorName)
addedVistorName := make([]string, 0)
for name, vistorCfg := range vistorCfgs {
if _, ok := ctl.vistorCfgs[name]; !ok {
ctl.vistorCfgs[name] = vistorCfg
addedVistorName = append(addedVistorName, name)
}
}
ctl.Info("vistor added: %v", addedVistorName)
}

View File

@@ -23,14 +23,15 @@ import (
"github.com/fatedier/frp/models/config"
"github.com/fatedier/frp/models/msg"
"github.com/fatedier/frp/models/proto/tcp"
"github.com/fatedier/frp/models/plugin"
"github.com/fatedier/frp/models/proto/udp"
"github.com/fatedier/frp/utils/errors"
frpIo "github.com/fatedier/frp/utils/io"
"github.com/fatedier/frp/utils/log"
frpNet "github.com/fatedier/frp/utils/net"
)
// Proxy defines how to work for different proxy type.
// Proxy defines how to deal with work connections for different proxy type.
type Proxy interface {
Run() error
@@ -66,6 +67,11 @@ func NewProxy(ctl *Control, pxyConf config.ProxyConf) (pxy Proxy) {
BaseProxy: baseProxy,
cfg: cfg,
}
case *config.StcpProxyConf:
pxy = &StcpProxy{
BaseProxy: baseProxy,
cfg: cfg,
}
}
return
}
@@ -81,57 +87,112 @@ type BaseProxy struct {
type TcpProxy struct {
BaseProxy
cfg *config.TcpProxyConf
cfg *config.TcpProxyConf
proxyPlugin plugin.Plugin
}
func (pxy *TcpProxy) Run() (err error) {
if pxy.cfg.Plugin != "" {
pxy.proxyPlugin, err = plugin.Create(pxy.cfg.Plugin, pxy.cfg.PluginParams)
if err != nil {
return
}
}
return
}
func (pxy *TcpProxy) Close() {
if pxy.proxyPlugin != nil {
pxy.proxyPlugin.Close()
}
}
func (pxy *TcpProxy) InWorkConn(conn frpNet.Conn) {
defer conn.Close()
HandleTcpWorkConnection(&pxy.cfg.LocalSvrConf, &pxy.cfg.BaseProxyConf, conn)
HandleTcpWorkConnection(&pxy.cfg.LocalSvrConf, pxy.proxyPlugin, &pxy.cfg.BaseProxyConf, conn)
}
// HTTP
type HttpProxy struct {
BaseProxy
cfg *config.HttpProxyConf
cfg *config.HttpProxyConf
proxyPlugin plugin.Plugin
}
func (pxy *HttpProxy) Run() (err error) {
if pxy.cfg.Plugin != "" {
pxy.proxyPlugin, err = plugin.Create(pxy.cfg.Plugin, pxy.cfg.PluginParams)
if err != nil {
return
}
}
return
}
func (pxy *HttpProxy) Close() {
if pxy.proxyPlugin != nil {
pxy.proxyPlugin.Close()
}
}
func (pxy *HttpProxy) InWorkConn(conn frpNet.Conn) {
defer conn.Close()
HandleTcpWorkConnection(&pxy.cfg.LocalSvrConf, &pxy.cfg.BaseProxyConf, conn)
HandleTcpWorkConnection(&pxy.cfg.LocalSvrConf, pxy.proxyPlugin, &pxy.cfg.BaseProxyConf, conn)
}
// HTTPS
type HttpsProxy struct {
BaseProxy
cfg *config.HttpsProxyConf
cfg *config.HttpsProxyConf
proxyPlugin plugin.Plugin
}
func (pxy *HttpsProxy) Run() (err error) {
if pxy.cfg.Plugin != "" {
pxy.proxyPlugin, err = plugin.Create(pxy.cfg.Plugin, pxy.cfg.PluginParams)
if err != nil {
return
}
}
return
}
func (pxy *HttpsProxy) Close() {
if pxy.proxyPlugin != nil {
pxy.proxyPlugin.Close()
}
}
func (pxy *HttpsProxy) InWorkConn(conn frpNet.Conn) {
defer conn.Close()
HandleTcpWorkConnection(&pxy.cfg.LocalSvrConf, &pxy.cfg.BaseProxyConf, conn)
HandleTcpWorkConnection(&pxy.cfg.LocalSvrConf, pxy.proxyPlugin, &pxy.cfg.BaseProxyConf, conn)
}
// STCP
type StcpProxy struct {
BaseProxy
cfg *config.StcpProxyConf
proxyPlugin plugin.Plugin
}
func (pxy *StcpProxy) Run() (err error) {
if pxy.cfg.Plugin != "" {
pxy.proxyPlugin, err = plugin.Create(pxy.cfg.Plugin, pxy.cfg.PluginParams)
if err != nil {
return
}
}
return
}
func (pxy *StcpProxy) Close() {
if pxy.proxyPlugin != nil {
pxy.proxyPlugin.Close()
}
}
func (pxy *StcpProxy) InWorkConn(conn frpNet.Conn) {
HandleTcpWorkConnection(&pxy.cfg.LocalSvrConf, pxy.proxyPlugin, &pxy.cfg.BaseProxyConf, conn)
}
// UDP
@@ -240,27 +301,41 @@ func (pxy *UdpProxy) InWorkConn(conn frpNet.Conn) {
}
// Common handler for tcp work connections.
func HandleTcpWorkConnection(localInfo *config.LocalSvrConf, baseInfo *config.BaseProxyConf, workConn frpNet.Conn) {
localConn, err := frpNet.ConnectTcpServer(fmt.Sprintf("%s:%d", localInfo.LocalIp, localInfo.LocalPort))
if err != nil {
workConn.Error("connect to local service [%s:%d] error: %v", localInfo.LocalIp, localInfo.LocalPort, err)
return
}
func HandleTcpWorkConnection(localInfo *config.LocalSvrConf, proxyPlugin plugin.Plugin,
baseInfo *config.BaseProxyConf, workConn frpNet.Conn) {
var remote io.ReadWriteCloser
var (
remote io.ReadWriteCloser
err error
)
remote = workConn
if baseInfo.UseEncryption {
remote, err = tcp.WithEncryption(remote, []byte(config.ClientCommonCfg.PrivilegeToken))
remote, err = frpIo.WithEncryption(remote, []byte(config.ClientCommonCfg.PrivilegeToken))
if err != nil {
workConn.Error("create encryption stream error: %v", err)
return
}
}
if baseInfo.UseCompression {
remote = tcp.WithCompression(remote)
remote = frpIo.WithCompression(remote)
}
if proxyPlugin != nil {
// if plugin is set, let plugin handle connections first
workConn.Debug("handle by plugin: %s", proxyPlugin.Name())
proxyPlugin.Handle(remote)
workConn.Debug("handle by plugin finished")
return
} else {
localConn, err := frpNet.ConnectServer("tcp", fmt.Sprintf("%s:%d", localInfo.LocalIp, localInfo.LocalPort))
if err != nil {
workConn.Error("connect to local service [%s:%d] error: %v", localInfo.LocalIp, localInfo.LocalPort, err)
return
}
workConn.Debug("join connections, localConn(l[%s] r[%s]) workConn(l[%s] r[%s])", localConn.LocalAddr().String(),
localConn.RemoteAddr().String(), workConn.LocalAddr().String(), workConn.RemoteAddr().String())
frpIo.Join(localConn, remote)
workConn.Debug("join connections closed")
}
workConn.Debug("join connections, localConn(l[%s] r[%s]) workConn(l[%s] r[%s])", localConn.LocalAddr().String(),
localConn.RemoteAddr().String(), workConn.LocalAddr().String(), workConn.RemoteAddr().String())
tcp.Join(localConn, remote)
workConn.Debug("join connections closed")
}

View File

@@ -14,7 +14,10 @@
package client
import "github.com/fatedier/frp/models/config"
import (
"github.com/fatedier/frp/models/config"
"github.com/fatedier/frp/utils/log"
)
type Service struct {
// manager control connection with server
@@ -23,11 +26,11 @@ type Service struct {
closedCh chan int
}
func NewService(pxyCfgs map[string]config.ProxyConf) (svr *Service) {
func NewService(pxyCfgs map[string]config.ProxyConf, vistorCfgs map[string]config.ProxyConf) (svr *Service) {
svr = &Service{
closedCh: make(chan int),
}
ctl := NewControl(svr, pxyCfgs)
ctl := NewControl(svr, pxyCfgs, vistorCfgs)
svr.ctl = ctl
return
}
@@ -38,6 +41,18 @@ func (svr *Service) Run() error {
return err
}
if config.ClientCommonCfg.AdminPort != 0 {
err = svr.RunAdminServer(config.ClientCommonCfg.AdminAddr, config.ClientCommonCfg.AdminPort)
if err != nil {
log.Warn("run admin server error: %v", err)
}
log.Info("admin server listen on %s:%d", config.ClientCommonCfg.AdminAddr, config.ClientCommonCfg.AdminPort)
}
<-svr.closedCh
return nil
}
func (svr *Service) Close() error {
return svr.ctl.Close()
}

145
client/vistor.go Normal file
View File

@@ -0,0 +1,145 @@
// Copyright 2017 fatedier, fatedier@gmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package client
import (
"io"
"sync"
"time"
"github.com/fatedier/frp/models/config"
"github.com/fatedier/frp/models/msg"
frpIo "github.com/fatedier/frp/utils/io"
"github.com/fatedier/frp/utils/log"
frpNet "github.com/fatedier/frp/utils/net"
"github.com/fatedier/frp/utils/util"
)
// Vistor is used for forward traffics from local port tot remote service.
type Vistor interface {
Run() error
Close()
log.Logger
}
func NewVistor(ctl *Control, pxyConf config.ProxyConf) (vistor Vistor) {
baseVistor := BaseVistor{
ctl: ctl,
Logger: log.NewPrefixLogger(pxyConf.GetName()),
}
switch cfg := pxyConf.(type) {
case *config.StcpProxyConf:
vistor = &StcpVistor{
BaseVistor: baseVistor,
cfg: cfg,
}
}
return
}
type BaseVistor struct {
ctl *Control
l frpNet.Listener
closed bool
mu sync.RWMutex
log.Logger
}
type StcpVistor struct {
BaseVistor
cfg *config.StcpProxyConf
}
func (sv *StcpVistor) Run() (err error) {
sv.l, err = frpNet.ListenTcp(sv.cfg.BindAddr, int64(sv.cfg.BindPort))
if err != nil {
return
}
go sv.worker()
return
}
func (sv *StcpVistor) Close() {
sv.l.Close()
}
func (sv *StcpVistor) worker() {
for {
conn, err := sv.l.Accept()
if err != nil {
sv.Warn("stcp local listener closed")
return
}
go sv.handleConn(conn)
}
}
func (sv *StcpVistor) handleConn(userConn frpNet.Conn) {
defer userConn.Close()
sv.Debug("get a new stcp user connection")
vistorConn, err := sv.ctl.connectServer()
if err != nil {
return
}
defer vistorConn.Close()
now := time.Now().Unix()
newVistorConnMsg := &msg.NewVistorConn{
ProxyName: sv.cfg.ServerName,
SignKey: util.GetAuthKey(sv.cfg.Sk, now),
Timestamp: now,
UseEncryption: sv.cfg.UseEncryption,
UseCompression: sv.cfg.UseCompression,
}
err = msg.WriteMsg(vistorConn, newVistorConnMsg)
if err != nil {
sv.Warn("send newVistorConnMsg to server error: %v", err)
return
}
var newVistorConnRespMsg msg.NewVistorConnResp
vistorConn.SetReadDeadline(time.Now().Add(10 * time.Second))
err = msg.ReadMsgInto(vistorConn, &newVistorConnRespMsg)
if err != nil {
sv.Warn("get newVistorConnRespMsg error: %v", err)
return
}
vistorConn.SetReadDeadline(time.Time{})
if newVistorConnRespMsg.Error != "" {
sv.Warn("start new vistor connection error: %s", newVistorConnRespMsg.Error)
return
}
var remote io.ReadWriteCloser
remote = vistorConn
if sv.cfg.UseEncryption {
remote, err = frpIo.WithEncryption(remote, []byte(sv.cfg.Sk))
if err != nil {
sv.Error("create encryption stream error: %v", err)
return
}
}
if sv.cfg.UseCompression {
remote = frpIo.WithCompression(remote)
}
frpIo.Join(userConn, remote)
}

View File

@@ -15,10 +15,17 @@
package main
import (
"encoding/base64"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
docopt "github.com/docopt/docopt-go"
ini "github.com/vaughan0/go-ini"
@@ -37,6 +44,7 @@ var usage string = `frpc is the client of frp
Usage:
frpc [-c config_file] [-L log_file] [--log-level=<log_level>] [--server-addr=<server_addr>]
frpc [-c config_file] --reload
frpc -h | --help
frpc -v | --version
@@ -45,13 +53,14 @@ Options:
-L log_file set output log file, including console
--log-level=<log_level> set log level: debug, info, warn, error
--server-addr=<server_addr> addr which frps is listening for, example: 0.0.0.0:7000
--reload reload configure file without program exit
-h --help show this screen
--version show version
-v --version show version
`
func main() {
var err error
confFile := "./frpc.ini"
confFile := "./frps.ini"
// the configures parsed from file will be replaced by those from command line if exist
args, err := docopt.Parse(usage, nil, true, version.Full(), false)
@@ -70,6 +79,47 @@ func main() {
fmt.Println(err)
os.Exit(1)
}
config.ClientCommonCfg.ConfigFile = confFile
// check if reload command
if args["--reload"] != nil {
if args["--reload"].(bool) {
req, err := http.NewRequest("GET", "http://"+
config.ClientCommonCfg.AdminAddr+":"+fmt.Sprintf("%d", config.ClientCommonCfg.AdminPort)+"/api/reload", nil)
if err != nil {
fmt.Printf("frps reload error: %v\n", err)
os.Exit(1)
}
authStr := "Basic " + base64.StdEncoding.EncodeToString([]byte(config.ClientCommonCfg.AdminUser+":"+
config.ClientCommonCfg.AdminPwd))
req.Header.Add("Authorization", authStr)
resp, err := http.DefaultClient.Do(req)
if err != nil {
fmt.Printf("frpc reload error: %v\n", err)
os.Exit(1)
} else {
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
fmt.Printf("frpc reload error: %v\n", err)
os.Exit(1)
}
res := &client.GeneralResponse{}
err = json.Unmarshal(body, &res)
if err != nil {
fmt.Printf("http response error: %s\n", strings.TrimSpace(string(body)))
os.Exit(1)
} else if res.Code != 0 {
fmt.Printf("reload error: %s\n", res.Msg)
os.Exit(1)
}
fmt.Printf("reload success\n")
os.Exit(0)
}
}
}
if args["-L"] != nil {
if args["-L"].(string) == "console" {
@@ -106,7 +156,7 @@ func main() {
}
}
pxyCfgs, err := config.LoadProxyConfFromFile(conf)
pxyCfgs, vistorCfgs, err := config.LoadProxyConfFromFile(config.ClientCommonCfg.User, conf, config.ClientCommonCfg.Start)
if err != nil {
fmt.Println(err)
os.Exit(1)
@@ -115,10 +165,25 @@ func main() {
log.InitLog(config.ClientCommonCfg.LogWay, config.ClientCommonCfg.LogFile,
config.ClientCommonCfg.LogLevel, config.ClientCommonCfg.LogMaxDays)
svr := client.NewService(pxyCfgs)
svr := client.NewService(pxyCfgs, vistorCfgs)
// Capture the exit signal if we use kcp.
if config.ClientCommonCfg.Protocol == "kcp" {
go HandleSignal(svr)
}
err = svr.Run()
if err != nil {
fmt.Println(err)
os.Exit(1)
}
}
func HandleSignal(svr *client.Service) {
ch := make(chan os.Signal)
signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
<-ch
svr.Close()
time.Sleep(250 * time.Millisecond)
os.Exit(0)
}

View File

@@ -1,83 +1,9 @@
# [common] is integral section
[common]
# A literal address or host name for IPv6 must be enclosed
# in square brackets, as in "[::1]:80", "[ipv6-host]:http" or "[ipv6-host%zone]:80"
server_addr = 0.0.0.0
server_addr = 127.0.0.1
server_port = 7000
# if you want to connect frps by http proxy, you can set http_proxy here or in global environment variables
# http_proxy = http://user:pwd@192.168.1.128:8080
# console or real logFile path like ./frpc.log
log_file = ./frpc.log
# trace, debug, info, warn, error
log_level = info
log_max_days = 3
# for authentication
privilege_token = 12345678
# connections will be established in advance, default value is zero
pool_count = 5
# if tcp stream multiplexing is used, default is true, it must be same with frps
tcp_mux = true
# your proxy name will be changed to {user}.{proxy}
user = your_name
# heartbeat configure, it's not recommended to modify the default value
# the default value of heartbeat_interval is 10 and heartbeat_timeout is 90
# heartbeat_interval = 30
# heartbeat_timeout = 90
# ssh is the proxy name same as server's configuration
# if user in [common] section is not empty, it will be changed to {user}.{proxy} such as your_name.ssh
[ssh]
# tcp | udp | http | https, default is tcp
type = tcp
local_ip = 127.0.0.1
local_port = 22
# true or false, if true, messages between frps and frpc will be encrypted, default is false
use_encryption = false
# if true, message will be compressed
use_compression = false
# remote port listen by frps
remote_port = 6001
[dns]
type = udp
local_ip = 114.114.114.114
local_port = 53
remote_port = 6002
use_encryption = false
use_compression = false
# Resolve your domain names to [server_addr] so you can use http://web01.yourdomain.com to browse web01 and http://web02.yourdomain.com to browse web02
[web01]
type = http
local_ip = 127.0.0.1
local_port = 80
use_encryption = false
use_compression = true
# http username and password are safety certification for http protocol
# if not set, you can access this custom_domains without certification
http_user = admin
http_pwd = admin
# if domain for frps is frps.com, then you can access [web01] proxy by URL http://test.frps.com
subdomain = web01
custom_domains = web02.yourdomain.com
# locations is only useful for http type
locations = /,/pic
host_header_rewrite = example.com
[web02]
type = https
local_ip = 127.0.0.1
local_port = 8000
use_encryption = false
use_compression = false
subdomain = web01
custom_domains = web02.yourdomain.com
remote_port = 6000

143
conf/frpc_full.ini Normal file
View File

@@ -0,0 +1,143 @@
# [common] is integral section
[common]
# A literal address or host name for IPv6 must be enclosed
# in square brackets, as in "[::1]:80", "[ipv6-host]:http" or "[ipv6-host%zone]:80"
server_addr = 0.0.0.0
server_port = 7000
# if you want to connect frps by http proxy, you can set http_proxy here or in global environment variables
# it only works when protocol is tcp
# http_proxy = http://user:pwd@192.168.1.128:8080
# console or real logFile path like ./frpc.log
log_file = ./frpc.log
# trace, debug, info, warn, error
log_level = info
log_max_days = 3
# for authentication
privilege_token = 12345678
# set admin address for control frpc's action by http api such as reload
admin_addr = 127.0.0.1
admin_port = 7400
admin_user = admin
admin_pwd = admin
# connections will be established in advance, default value is zero
pool_count = 5
# if tcp stream multiplexing is used, default is true, it must be same with frps
tcp_mux = true
# your proxy name will be changed to {user}.{proxy}
user = your_name
# decide if exit program when first login failed, otherwise continuous relogin to frps
# default is true
login_fail_exit = true
# communication protocol used to connect to server
# now it supports tcp and kcp, default is tcp
protocol = tcp
# proxy names you want to start divided by ','
# default is empty, means all proxies
# start = ssh,dns
# heartbeat configure, it's not recommended to modify the default value
# the default value of heartbeat_interval is 10 and heartbeat_timeout is 90
# heartbeat_interval = 30
# heartbeat_timeout = 90
# ssh is the proxy name same as server's configuration
# if user in [common] section is not empty, it will be changed to {user}.{proxy} such as your_name.ssh
[ssh]
# tcp | udp | http | https, default is tcp
type = tcp
local_ip = 127.0.0.1
local_port = 22
# true or false, if true, messages between frps and frpc will be encrypted, default is false
use_encryption = false
# if true, message will be compressed
use_compression = false
# remote port listen by frps
remote_port = 6001
[dns]
type = udp
local_ip = 114.114.114.114
local_port = 53
remote_port = 6002
use_encryption = false
use_compression = false
# Resolve your domain names to [server_addr] so you can use http://web01.yourdomain.com to browse web01 and http://web02.yourdomain.com to browse web02
[web01]
type = http
local_ip = 127.0.0.1
local_port = 80
use_encryption = false
use_compression = true
# http username and password are safety certification for http protocol
# if not set, you can access this custom_domains without certification
http_user = admin
http_pwd = admin
# if domain for frps is frps.com, then you can access [web01] proxy by URL http://test.frps.com
subdomain = web01
custom_domains = web02.yourdomain.com
# locations is only useful for http type
locations = /,/pic
host_header_rewrite = example.com
[web02]
type = https
local_ip = 127.0.0.1
local_port = 8000
use_encryption = false
use_compression = false
subdomain = web01
custom_domains = web02.yourdomain.com
[plugin_unix_domain_socket]
type = tcp
remote_port = 6003
# if plugin is defined, local_ip and local_port is useless
# plugin will handle connections got from frps
plugin = unix_domain_socket
# params set with prefix "plugin_" that plugin needed
plugin_unix_path = /var/run/docker.sock
[plugin_http_proxy]
type = tcp
remote_port = 6004
plugin = http_proxy
plugin_http_user = abc
plugin_http_passwd = abc
[secret_tcp]
# If the type is secret tcp, remote_port is useless
# Who want to connect local port should deploy another frpc with stcp proxy and role is vistor
type = stcp
# sk used for authentication for vistors
sk = abcdefg
local_ip = 127.0.0.1
local_port = 22
use_encryption = false
use_compression = false
# user of frpc should be same in both stcp server and stcp vistor
[secret_tcp_vistor]
# frpc role vistor -> frps -> frpc role server
role = vistor
type = stcp
# the server name you want to vistor
server_name = secret_tcp
sk = abcdefg
# connect this address to vistor stcp server
bind_addr = 127.0.0.1
bind_port = 9000
use_encryption = false
use_compression = false

View File

@@ -1,10 +0,0 @@
[common]
server_addr = 0.0.0.0
server_port = 7000
#privilege_token = 12345678
[ssh]
type = tcp
local_ip = 127.0.0.1
local_port = 22
remote_port = 6000

View File

@@ -1,51 +1,2 @@
# [common] is integral section
[common]
# A literal address or host name for IPv6 must be enclosed
# in square brackets, as in "[::1]:80", "[ipv6-host]:http" or "[ipv6-host%zone]:80"
bind_addr = 0.0.0.0
bind_port = 7000
# if you want to support virtual host, you must set the http port for listening (optional)
vhost_http_port = 80
vhost_https_port = 443
# if you want to configure or reload frps by dashboard, dashboard_port must be set
dashboard_port = 7500
# dashboard user and pwd for basic auth protect, if not set, both default value is admin
dashboard_user = admin
dashboard_pwd = admin
# dashboard assets directory(only for debug mode)
# assets_dir = ./static
# console or real logFile path like ./frps.log
log_file = ./frps.log
# trace, debug, info, warn, error
log_level = info
log_max_days = 3
# privilege mode is the only supported mode since v0.10.0
privilege_token = 12345678
# heartbeat configure, it's not recommended to modify the default value
# the default value of heartbeat_timeout is 90
# heartbeat_timeout = 90
# only allow frpc to bind ports you list, if you set nothing, there won't be any limit
privilege_allow_ports = 2000-3000,3001,3003,4000-50000
# pool_count in each proxy will change to max_pool_count if they exceed the maximum value
max_pool_count = 5
# authentication_timeout means the timeout interval (seconds) when the frpc connects frps
# if authentication_timeout is zero, the time is not verified, default is 900s
authentication_timeout = 900
# if subdomain_host is not empty, you can set subdomain when type is http or https in frpc's configure file
# when subdomain is test, the host used by routing is test.frps.com
subdomain_host = frps.com
# if tcp stream multiplexing is used, default is true
tcp_mux = true

58
conf/frps_full.ini Normal file
View File

@@ -0,0 +1,58 @@
# [common] is integral section
[common]
# A literal address or host name for IPv6 must be enclosed
# in square brackets, as in "[::1]:80", "[ipv6-host]:http" or "[ipv6-host%zone]:80"
bind_addr = 0.0.0.0
bind_port = 7000
# udp port used for kcp protocol, it can be same with 'bind_port'
# if not set, kcp is disabled in frps
kcp_bind_port = 7000
# specify which address proxy will listen for, default value is same with bind_addr
# proxy_bind_addr = 127.0.0.1
# if you want to support virtual host, you must set the http port for listening (optional)
vhost_http_port = 80
vhost_https_port = 443
# set dashboard_port to view dashboard of frps
dashboard_port = 7500
# dashboard user and pwd for basic auth protect, if not set, both default value is admin
dashboard_user = admin
dashboard_pwd = admin
# dashboard assets directory(only for debug mode)
# assets_dir = ./static
# console or real logFile path like ./frps.log
log_file = ./frps.log
# trace, debug, info, warn, error
log_level = info
log_max_days = 3
# privilege mode is the only supported mode since v0.10.0
privilege_token = 12345678
# heartbeat configure, it's not recommended to modify the default value
# the default value of heartbeat_timeout is 90
# heartbeat_timeout = 90
# only allow frpc to bind ports you list, if you set nothing, there won't be any limit
privilege_allow_ports = 2000-3000,3001,3003,4000-50000
# pool_count in each proxy will change to max_pool_count if they exceed the maximum value
max_pool_count = 5
# authentication_timeout means the timeout interval (seconds) when the frpc connects frps
# if authentication_timeout is zero, the time is not verified, default is 900s
authentication_timeout = 900
# if subdomain_host is not empty, you can set subdomain when type is http or https in frpc's configure file
# when subdomain is test, the host used by routing is test.frps.com
subdomain_host = frps.com
# if tcp stream multiplexing is used, default is true
tcp_mux = true

View File

@@ -1,7 +0,0 @@
[common]
bind_addr = 0.0.0.0
bind_port = 7000
vhost_http_port = 80
vhost_https_port = 443
dashboard_port = 7500
#privilege_token = 12345678

Binary file not shown.

After

Width:  |  Height:  |  Size: 27 KiB

View File

@@ -18,6 +18,7 @@ import (
"fmt"
"os"
"strconv"
"strings"
ini "github.com/vaughan0/go-ini"
)
@@ -35,9 +36,16 @@ type ClientCommonConf struct {
LogLevel string
LogMaxDays int64
PrivilegeToken string
AdminAddr string
AdminPort int64
AdminUser string
AdminPwd string
PoolCount int
TcpMux bool
User string
LoginFailExit bool
Start map[string]struct{}
Protocol string
HeartBeatInterval int64
HeartBeatTimeout int64
}
@@ -53,9 +61,16 @@ func GetDeaultClientCommonConf() *ClientCommonConf {
LogLevel: "info",
LogMaxDays: 3,
PrivilegeToken: "",
AdminAddr: "127.0.0.1",
AdminPort: 0,
AdminUser: "",
AdminPwd: "",
PoolCount: 1,
TcpMux: true,
User: "",
LoginFailExit: true,
Start: make(map[string]struct{}),
Protocol: "tcp",
HeartBeatInterval: 30,
HeartBeatTimeout: 90,
}
@@ -104,7 +119,9 @@ func LoadClientCommonConf(conf ini.File) (cfg *ClientCommonConf, err error) {
tmpStr, ok = conf.Get("common", "log_max_days")
if ok {
cfg.LogMaxDays, _ = strconv.ParseInt(tmpStr, 10, 64)
if v, err = strconv.ParseInt(tmpStr, 10, 64); err == nil {
cfg.LogMaxDays = v
}
}
tmpStr, ok = conf.Get("common", "privilege_token")
@@ -112,6 +129,28 @@ func LoadClientCommonConf(conf ini.File) (cfg *ClientCommonConf, err error) {
cfg.PrivilegeToken = tmpStr
}
tmpStr, ok = conf.Get("common", "admin_addr")
if ok {
cfg.AdminAddr = tmpStr
}
tmpStr, ok = conf.Get("common", "admin_port")
if ok {
if v, err = strconv.ParseInt(tmpStr, 10, 64); err == nil {
cfg.AdminPort = v
}
}
tmpStr, ok = conf.Get("common", "admin_user")
if ok {
cfg.AdminUser = tmpStr
}
tmpStr, ok = conf.Get("common", "admin_pwd")
if ok {
cfg.AdminPwd = tmpStr
}
tmpStr, ok = conf.Get("common", "pool_count")
if ok {
v, err = strconv.ParseInt(tmpStr, 10, 64)
@@ -134,6 +173,30 @@ func LoadClientCommonConf(conf ini.File) (cfg *ClientCommonConf, err error) {
cfg.User = tmpStr
}
tmpStr, ok = conf.Get("common", "start")
if ok {
proxyNames := strings.Split(tmpStr, ",")
for _, name := range proxyNames {
cfg.Start[strings.TrimSpace(name)] = struct{}{}
}
}
tmpStr, ok = conf.Get("common", "login_fail_exit")
if ok && tmpStr == "false" {
cfg.LoginFailExit = false
} else {
cfg.LoginFailExit = true
}
tmpStr, ok = conf.Get("common", "protocol")
if ok {
// Now it only support tcp and kcp.
if tmpStr != "kcp" {
tmpStr = "tcp"
}
cfg.Protocol = tmpStr
}
tmpStr, ok = conf.Get("common", "heartbeat_timeout")
if ok {
v, err = strconv.ParseInt(tmpStr, 10, 64)

View File

@@ -35,6 +35,7 @@ func init() {
proxyConfTypeMap[consts.UdpProxy] = reflect.TypeOf(UdpProxyConf{})
proxyConfTypeMap[consts.HttpProxy] = reflect.TypeOf(HttpProxyConf{})
proxyConfTypeMap[consts.HttpsProxy] = reflect.TypeOf(HttpsProxyConf{})
proxyConfTypeMap[consts.StcpProxy] = reflect.TypeOf(StcpProxyConf{})
}
// NewConfByType creates a empty ProxyConf object by proxyType.
@@ -55,6 +56,7 @@ type ProxyConf interface {
LoadFromFile(name string, conf ini.Section) error
UnMarshalToMsg(pMsg *msg.NewProxy)
Check() error
Compare(conf ProxyConf) bool
}
func NewProxyConf(pMsg *msg.NewProxy) (cfg ProxyConf, err error) {
@@ -104,6 +106,16 @@ func (cfg *BaseProxyConf) GetBaseInfo() *BaseProxyConf {
return cfg
}
func (cfg *BaseProxyConf) compare(cmp *BaseProxyConf) bool {
if cfg.ProxyName != cmp.ProxyName ||
cfg.ProxyType != cmp.ProxyType ||
cfg.UseEncryption != cmp.UseEncryption ||
cfg.UseCompression != cmp.UseCompression {
return false
}
return true
}
func (cfg *BaseProxyConf) LoadFromMsg(pMsg *msg.NewProxy) {
cfg.ProxyName = pMsg.ProxyName
cfg.ProxyType = pMsg.ProxyType
@@ -148,8 +160,16 @@ type BindInfoConf struct {
RemotePort int64 `json:"remote_port"`
}
func (cfg *BindInfoConf) compare(cmp *BindInfoConf) bool {
if cfg.BindAddr != cmp.BindAddr ||
cfg.RemotePort != cmp.RemotePort {
return false
}
return true
}
func (cfg *BindInfoConf) LoadFromMsg(pMsg *msg.NewProxy) {
cfg.BindAddr = ServerCommonCfg.BindAddr
cfg.BindAddr = ServerCommonCfg.ProxyBindAddr
cfg.RemotePort = pMsg.RemotePort
}
@@ -187,6 +207,14 @@ type DomainConf struct {
SubDomain string `json:"sub_domain"`
}
func (cfg *DomainConf) compare(cmp *DomainConf) bool {
if strings.Join(cfg.CustomDomains, " ") != strings.Join(cmp.CustomDomains, " ") ||
cfg.SubDomain != cmp.SubDomain {
return false
}
return true
}
func (cfg *DomainConf) LoadFromMsg(pMsg *msg.NewProxy) {
cfg.CustomDomains = pMsg.CustomDomains
cfg.SubDomain = pMsg.SubDomain
@@ -239,11 +267,20 @@ func (cfg *DomainConf) check() (err error) {
return nil
}
// Local service info
type LocalSvrConf struct {
LocalIp string `json:"-"`
LocalPort int `json:"-"`
}
func (cfg *LocalSvrConf) compare(cmp *LocalSvrConf) bool {
if cfg.LocalIp != cmp.LocalIp ||
cfg.LocalPort != cmp.LocalPort {
return false
}
return true
}
func (cfg *LocalSvrConf) LoadFromFile(name string, section ini.Section) (err error) {
if cfg.LocalIp = section["local_ip"]; cfg.LocalIp == "" {
cfg.LocalIp = "127.0.0.1"
@@ -259,12 +296,63 @@ func (cfg *LocalSvrConf) LoadFromFile(name string, section ini.Section) (err err
return nil
}
type PluginConf struct {
Plugin string `json:"-"`
PluginParams map[string]string `json:"-"`
}
func (cfg *PluginConf) compare(cmp *PluginConf) bool {
if cfg.Plugin != cmp.Plugin ||
len(cfg.PluginParams) != len(cmp.PluginParams) {
return false
}
for k, v := range cfg.PluginParams {
value, ok := cmp.PluginParams[k]
if !ok || v != value {
return false
}
}
return true
}
func (cfg *PluginConf) LoadFromFile(name string, section ini.Section) (err error) {
cfg.Plugin = section["plugin"]
cfg.PluginParams = make(map[string]string)
if cfg.Plugin != "" {
// get params begin with "plugin_"
for k, v := range section {
if strings.HasPrefix(k, "plugin_") {
cfg.PluginParams[k] = v
}
}
} else {
return fmt.Errorf("Parse conf error: proxy [%s] no plugin info found", name)
}
return
}
// TCP
type TcpProxyConf struct {
BaseProxyConf
BindInfoConf
LocalSvrConf
PluginConf
}
func (cfg *TcpProxyConf) Compare(cmp ProxyConf) bool {
cmpConf, ok := cmp.(*TcpProxyConf)
if !ok {
return false
}
if !cfg.BaseProxyConf.compare(&cmpConf.BaseProxyConf) ||
!cfg.BindInfoConf.compare(&cmpConf.BindInfoConf) ||
!cfg.LocalSvrConf.compare(&cmpConf.LocalSvrConf) ||
!cfg.PluginConf.compare(&cmpConf.PluginConf) {
return false
}
return true
}
func (cfg *TcpProxyConf) LoadFromMsg(pMsg *msg.NewProxy) {
@@ -279,8 +367,11 @@ func (cfg *TcpProxyConf) LoadFromFile(name string, section ini.Section) (err err
if err = cfg.BindInfoConf.LoadFromFile(name, section); err != nil {
return
}
if err = cfg.LocalSvrConf.LoadFromFile(name, section); err != nil {
return
if err = cfg.PluginConf.LoadFromFile(name, section); err != nil {
if err = cfg.LocalSvrConf.LoadFromFile(name, section); err != nil {
return
}
}
return
}
@@ -303,6 +394,20 @@ type UdpProxyConf struct {
LocalSvrConf
}
func (cfg *UdpProxyConf) Compare(cmp ProxyConf) bool {
cmpConf, ok := cmp.(*UdpProxyConf)
if !ok {
return false
}
if !cfg.BaseProxyConf.compare(&cmpConf.BaseProxyConf) ||
!cfg.BindInfoConf.compare(&cmpConf.BindInfoConf) ||
!cfg.LocalSvrConf.compare(&cmpConf.LocalSvrConf) {
return false
}
return true
}
func (cfg *UdpProxyConf) LoadFromMsg(pMsg *msg.NewProxy) {
cfg.BaseProxyConf.LoadFromMsg(pMsg)
cfg.BindInfoConf.LoadFromMsg(pMsg)
@@ -337,6 +442,7 @@ type HttpProxyConf struct {
DomainConf
LocalSvrConf
PluginConf
Locations []string `json:"locations"`
HostHeaderRewrite string `json:"host_header_rewrite"`
@@ -344,6 +450,25 @@ type HttpProxyConf struct {
HttpPwd string `json:"-"`
}
func (cfg *HttpProxyConf) Compare(cmp ProxyConf) bool {
cmpConf, ok := cmp.(*HttpProxyConf)
if !ok {
return false
}
if !cfg.BaseProxyConf.compare(&cmpConf.BaseProxyConf) ||
!cfg.DomainConf.compare(&cmpConf.DomainConf) ||
!cfg.LocalSvrConf.compare(&cmpConf.LocalSvrConf) ||
!cfg.PluginConf.compare(&cmpConf.PluginConf) ||
strings.Join(cfg.Locations, " ") != strings.Join(cmpConf.Locations, " ") ||
cfg.HostHeaderRewrite != cmpConf.HostHeaderRewrite ||
cfg.HttpUser != cmpConf.HttpUser ||
cfg.HttpPwd != cmpConf.HttpPwd {
return false
}
return true
}
func (cfg *HttpProxyConf) LoadFromMsg(pMsg *msg.NewProxy) {
cfg.BaseProxyConf.LoadFromMsg(pMsg)
cfg.DomainConf.LoadFromMsg(pMsg)
@@ -361,8 +486,10 @@ func (cfg *HttpProxyConf) LoadFromFile(name string, section ini.Section) (err er
if err = cfg.DomainConf.LoadFromFile(name, section); err != nil {
return
}
if err = cfg.LocalSvrConf.LoadFromFile(name, section); err != nil {
return
if err = cfg.PluginConf.LoadFromFile(name, section); err != nil {
if err = cfg.LocalSvrConf.LoadFromFile(name, section); err != nil {
return
}
}
var (
@@ -405,6 +532,22 @@ type HttpsProxyConf struct {
DomainConf
LocalSvrConf
PluginConf
}
func (cfg *HttpsProxyConf) Compare(cmp ProxyConf) bool {
cmpConf, ok := cmp.(*HttpsProxyConf)
if !ok {
return false
}
if !cfg.BaseProxyConf.compare(&cmpConf.BaseProxyConf) ||
!cfg.DomainConf.compare(&cmpConf.DomainConf) ||
!cfg.LocalSvrConf.compare(&cmpConf.LocalSvrConf) ||
!cfg.PluginConf.compare(&cmpConf.PluginConf) {
return false
}
return true
}
func (cfg *HttpsProxyConf) LoadFromMsg(pMsg *msg.NewProxy) {
@@ -419,8 +562,10 @@ func (cfg *HttpsProxyConf) LoadFromFile(name string, section ini.Section) (err e
if err = cfg.DomainConf.LoadFromFile(name, section); err != nil {
return
}
if err = cfg.LocalSvrConf.LoadFromFile(name, section); err != nil {
return
if err = cfg.PluginConf.LoadFromFile(name, section); err != nil {
if err = cfg.LocalSvrConf.LoadFromFile(name, section); err != nil {
return
}
}
return
}
@@ -438,19 +583,126 @@ func (cfg *HttpsProxyConf) Check() (err error) {
return
}
func LoadProxyConfFromFile(conf ini.File) (proxyConfs map[string]ProxyConf, err error) {
var prefix string
if ClientCommonCfg.User != "" {
prefix = ClientCommonCfg.User + "."
// STCP
type StcpProxyConf struct {
BaseProxyConf
Role string `json:"role"`
Sk string `json:"sk"`
// used in role server
LocalSvrConf
PluginConf
// used in role vistor
ServerName string `json:"server_name"`
BindAddr string `json:"bind_addr"`
BindPort int `json:"bind_port"`
}
func (cfg *StcpProxyConf) Compare(cmp ProxyConf) bool {
cmpConf, ok := cmp.(*StcpProxyConf)
if !ok {
return false
}
proxyConfs = make(map[string]ProxyConf)
for name, section := range conf {
if name != "common" {
cfg, err := NewProxyConfFromFile(name, section)
if err != nil {
return proxyConfs, err
if !cfg.BaseProxyConf.compare(&cmpConf.BaseProxyConf) ||
!cfg.LocalSvrConf.compare(&cmpConf.LocalSvrConf) ||
!cfg.PluginConf.compare(&cmpConf.PluginConf) ||
cfg.Role != cmpConf.Role ||
cfg.Sk != cmpConf.Sk ||
cfg.ServerName != cmpConf.ServerName ||
cfg.BindAddr != cmpConf.BindAddr ||
cfg.BindPort != cmpConf.BindPort {
return false
}
return true
}
// Only for role server.
func (cfg *StcpProxyConf) LoadFromMsg(pMsg *msg.NewProxy) {
cfg.BaseProxyConf.LoadFromMsg(pMsg)
cfg.Sk = pMsg.Sk
}
func (cfg *StcpProxyConf) LoadFromFile(name string, section ini.Section) (err error) {
if err = cfg.BaseProxyConf.LoadFromFile(name, section); err != nil {
return
}
tmpStr := section["role"]
if tmpStr == "server" || tmpStr == "vistor" {
cfg.Role = tmpStr
} else {
cfg.Role = "server"
}
cfg.Sk = section["sk"]
if tmpStr == "vistor" {
prefix := section["prefix"]
cfg.ServerName = prefix + section["server_name"]
if cfg.BindAddr = section["bind_addr"]; cfg.BindAddr == "" {
cfg.BindAddr = "127.0.0.1"
}
if tmpStr, ok := section["bind_port"]; ok {
if cfg.BindPort, err = strconv.Atoi(tmpStr); err != nil {
return fmt.Errorf("Parse conf error: proxy [%s] bind_port error", name)
}
} else {
return fmt.Errorf("Parse conf error: proxy [%s] bind_port not found", name)
}
} else {
if err = cfg.PluginConf.LoadFromFile(name, section); err != nil {
if err = cfg.LocalSvrConf.LoadFromFile(name, section); err != nil {
return
}
}
}
return
}
func (cfg *StcpProxyConf) UnMarshalToMsg(pMsg *msg.NewProxy) {
cfg.BaseProxyConf.UnMarshalToMsg(pMsg)
pMsg.Sk = cfg.Sk
}
func (cfg *StcpProxyConf) Check() (err error) {
return
}
// if len(startProxy) is 0, start all
// otherwise just start proxies in startProxy map
func LoadProxyConfFromFile(prefix string, conf ini.File, startProxy map[string]struct{}) (
proxyConfs map[string]ProxyConf, vistorConfs map[string]ProxyConf, err error) {
if prefix != "" {
prefix += "."
}
startAll := true
if len(startProxy) > 0 {
startAll = false
}
proxyConfs = make(map[string]ProxyConf)
vistorConfs = make(map[string]ProxyConf)
for name, section := range conf {
_, shouldStart := startProxy[name]
if name != "common" && (startAll || shouldStart) {
// some proxy or visotr configure may be used this prefix
section["prefix"] = prefix
cfg, err := NewProxyConfFromFile(name, section)
if err != nil {
return proxyConfs, vistorConfs, err
}
role := section["role"]
if role == "vistor" {
vistorConfs[prefix+name] = cfg
} else {
proxyConfs[prefix+name] = cfg
}
proxyConfs[prefix+name] = cfg
}
}
return

View File

@@ -27,9 +27,11 @@ var ServerCommonCfg *ServerCommonConf
// common config
type ServerCommonConf struct {
ConfigFile string
BindAddr string
BindPort int64
ConfigFile string
BindAddr string
BindPort int64
KcpBindPort int64
ProxyBindAddr string
// If VhostHttpPort equals 0, don't listen a public port for http protocol.
VhostHttpPort int64
@@ -64,6 +66,8 @@ func GetDefaultServerCommonConf() *ServerCommonConf {
ConfigFile: "./frps.ini",
BindAddr: "0.0.0.0",
BindPort: 7000,
KcpBindPort: 0,
ProxyBindAddr: "0.0.0.0",
VhostHttpPort: 0,
VhostHttpsPort: 0,
DashboardPort: 0,
@@ -107,6 +111,21 @@ func LoadServerCommonConf(conf ini.File) (cfg *ServerCommonConf, err error) {
}
}
tmpStr, ok = conf.Get("common", "kcp_bind_port")
if ok {
v, err = strconv.ParseInt(tmpStr, 10, 64)
if err == nil && v > 0 {
cfg.KcpBindPort = v
}
}
tmpStr, ok = conf.Get("common", "proxy_bind_addr")
if ok {
cfg.ProxyBindAddr = tmpStr
} else {
cfg.ProxyBindAddr = cfg.BindAddr
}
tmpStr, ok = conf.Get("common", "vhost_http_port")
if ok {
cfg.VhostHttpPort, err = strconv.ParseInt(tmpStr, 10, 64)

View File

@@ -27,4 +27,5 @@ var (
UdpProxy string = "udp"
HttpProxy string = "http"
HttpsProxy string = "https"
StcpProxy string = "stcp"
)

View File

@@ -20,16 +20,19 @@ import (
)
const (
TypeLogin = 'o'
TypeLoginResp = '1'
TypeNewProxy = 'p'
TypeNewProxyResp = '2'
TypeNewWorkConn = 'w'
TypeReqWorkConn = 'r'
TypeStartWorkConn = 's'
TypePing = 'h'
TypePong = '4'
TypeUdpPacket = 'u'
TypeLogin = 'o'
TypeLoginResp = '1'
TypeNewProxy = 'p'
TypeNewProxyResp = '2'
TypeCloseProxy = 'c'
TypeNewWorkConn = 'w'
TypeReqWorkConn = 'r'
TypeStartWorkConn = 's'
TypeNewVistorConn = 'v'
TypeNewVistorConnResp = '3'
TypePing = 'h'
TypePong = '4'
TypeUdpPacket = 'u'
)
var (
@@ -45,9 +48,12 @@ func init() {
TypeMap[TypeLoginResp] = reflect.TypeOf(LoginResp{})
TypeMap[TypeNewProxy] = reflect.TypeOf(NewProxy{})
TypeMap[TypeNewProxyResp] = reflect.TypeOf(NewProxyResp{})
TypeMap[TypeCloseProxy] = reflect.TypeOf(CloseProxy{})
TypeMap[TypeNewWorkConn] = reflect.TypeOf(NewWorkConn{})
TypeMap[TypeReqWorkConn] = reflect.TypeOf(ReqWorkConn{})
TypeMap[TypeStartWorkConn] = reflect.TypeOf(StartWorkConn{})
TypeMap[TypeNewVistorConn] = reflect.TypeOf(NewVistorConn{})
TypeMap[TypeNewVistorConnResp] = reflect.TypeOf(NewVistorConnResp{})
TypeMap[TypePing] = reflect.TypeOf(Ping{})
TypeMap[TypePong] = reflect.TypeOf(Pong{})
TypeMap[TypeUdpPacket] = reflect.TypeOf(UdpPacket{})
@@ -98,6 +104,9 @@ type NewProxy struct {
HostHeaderRewrite string `json:"host_header_rewrite"`
HttpUser string `json:"http_user"`
HttpPwd string `json:"http_pwd"`
// stcp
Sk string `json:"sk"`
}
type NewProxyResp struct {
@@ -105,6 +114,10 @@ type NewProxyResp struct {
Error string `json:"error"`
}
type CloseProxy struct {
ProxyName string `json:"proxy_name"`
}
type NewWorkConn struct {
RunId string `json:"run_id"`
}
@@ -116,6 +129,19 @@ type StartWorkConn struct {
ProxyName string `json:"proxy_name"`
}
type NewVistorConn struct {
ProxyName string `json:"proxy_name"`
SignKey string `json:"sign_key"`
Timestamp int64 `json:"timestamp"`
UseEncryption bool `json:"use_encryption"`
UseCompression bool `json:"use_compression"`
}
type NewVistorConnResp struct {
ProxyName string `json:"proxy_name"`
Error string `json:"error"`
}
type Ping struct {
}

283
models/plugin/http_proxy.go Normal file
View File

@@ -0,0 +1,283 @@
// Copyright 2017 frp team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package plugin
import (
"bufio"
"encoding/base64"
"fmt"
"io"
"net"
"net/http"
"strings"
"sync"
"github.com/fatedier/frp/utils/errors"
frpIo "github.com/fatedier/frp/utils/io"
frpNet "github.com/fatedier/frp/utils/net"
)
const PluginHttpProxy = "http_proxy"
func init() {
Register(PluginHttpProxy, NewHttpProxyPlugin)
}
type Listener struct {
conns chan net.Conn
closed bool
mu sync.Mutex
}
func NewProxyListener() *Listener {
return &Listener{
conns: make(chan net.Conn, 64),
}
}
func (l *Listener) Accept() (net.Conn, error) {
conn, ok := <-l.conns
if !ok {
return nil, fmt.Errorf("listener closed")
}
return conn, nil
}
func (l *Listener) PutConn(conn net.Conn) error {
err := errors.PanicToError(func() {
l.conns <- conn
})
return err
}
func (l *Listener) Close() error {
l.mu.Lock()
defer l.mu.Unlock()
if !l.closed {
close(l.conns)
l.closed = true
}
return nil
}
func (l *Listener) Addr() net.Addr {
return (*net.TCPAddr)(nil)
}
type HttpProxy struct {
l *Listener
s *http.Server
AuthUser string
AuthPasswd string
}
func NewHttpProxyPlugin(params map[string]string) (Plugin, error) {
user := params["plugin_http_user"]
passwd := params["plugin_http_passwd"]
listener := NewProxyListener()
hp := &HttpProxy{
l: listener,
AuthUser: user,
AuthPasswd: passwd,
}
hp.s = &http.Server{
Handler: hp,
}
go hp.s.Serve(listener)
return hp, nil
}
func (hp *HttpProxy) Name() string {
return PluginHttpProxy
}
func (hp *HttpProxy) Handle(conn io.ReadWriteCloser) {
var wrapConn frpNet.Conn
if realConn, ok := conn.(frpNet.Conn); ok {
wrapConn = realConn
} else {
wrapConn = frpNet.WrapReadWriteCloserToConn(conn)
}
sc, rd := frpNet.NewShareConn(wrapConn)
request, err := http.ReadRequest(bufio.NewReader(rd))
if err != nil {
wrapConn.Close()
return
}
if request.Method == http.MethodConnect {
hp.handleConnectReq(request, frpIo.WrapReadWriteCloser(rd, wrapConn, nil))
return
}
hp.l.PutConn(sc)
return
}
func (hp *HttpProxy) Close() error {
hp.s.Close()
hp.l.Close()
return nil
}
func (hp *HttpProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
if ok := hp.Auth(req); !ok {
rw.Header().Set("Proxy-Authenticate", "Basic")
rw.WriteHeader(http.StatusProxyAuthRequired)
return
}
if req.Method == http.MethodConnect {
// deprecated
// Connect request is handled in Handle function.
hp.ConnectHandler(rw, req)
} else {
hp.HttpHandler(rw, req)
}
}
func (hp *HttpProxy) HttpHandler(rw http.ResponseWriter, req *http.Request) {
removeProxyHeaders(req)
resp, err := http.DefaultTransport.RoundTrip(req)
if err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
return
}
defer resp.Body.Close()
copyHeaders(rw.Header(), resp.Header)
rw.WriteHeader(resp.StatusCode)
_, err = io.Copy(rw, resp.Body)
if err != nil && err != io.EOF {
return
}
}
// deprecated
// Hijack needs to SetReadDeadline on the Conn of the request, but if we use stream compression here,
// we may always get i/o timeout error.
func (hp *HttpProxy) ConnectHandler(rw http.ResponseWriter, req *http.Request) {
hj, ok := rw.(http.Hijacker)
if !ok {
rw.WriteHeader(http.StatusInternalServerError)
return
}
client, _, err := hj.Hijack()
if err != nil {
rw.WriteHeader(http.StatusInternalServerError)
return
}
remote, err := net.Dial("tcp", req.URL.Host)
if err != nil {
http.Error(rw, "Failed", http.StatusBadRequest)
client.Close()
return
}
client.Write([]byte("HTTP/1.1 200 OK\r\n\r\n"))
go frpIo.Join(remote, client)
}
func (hp *HttpProxy) Auth(req *http.Request) bool {
if hp.AuthUser == "" && hp.AuthPasswd == "" {
return true
}
s := strings.SplitN(req.Header.Get("Proxy-Authorization"), " ", 2)
if len(s) != 2 {
return false
}
b, err := base64.StdEncoding.DecodeString(s[1])
if err != nil {
return false
}
pair := strings.SplitN(string(b), ":", 2)
if len(pair) != 2 {
return false
}
if pair[0] != hp.AuthUser || pair[1] != hp.AuthPasswd {
return false
}
return true
}
func (hp *HttpProxy) handleConnectReq(req *http.Request, rwc io.ReadWriteCloser) {
defer rwc.Close()
if ok := hp.Auth(req); !ok {
res := getBadResponse()
res.Write(rwc)
return
}
remote, err := net.Dial("tcp", req.URL.Host)
if err != nil {
res := &http.Response{
StatusCode: 400,
Proto: "HTTP/1.1",
ProtoMajor: 1,
ProtoMinor: 1,
}
res.Write(rwc)
return
}
rwc.Write([]byte("HTTP/1.1 200 OK\r\n\r\n"))
frpIo.Join(remote, rwc)
}
func copyHeaders(dst, src http.Header) {
for key, values := range src {
for _, value := range values {
dst.Add(key, value)
}
}
}
func removeProxyHeaders(req *http.Request) {
req.RequestURI = ""
req.Header.Del("Proxy-Connection")
req.Header.Del("Connection")
req.Header.Del("Proxy-Authenticate")
req.Header.Del("Proxy-Authorization")
req.Header.Del("TE")
req.Header.Del("Trailers")
req.Header.Del("Transfer-Encoding")
req.Header.Del("Upgrade")
}
func getBadResponse() *http.Response {
header := make(map[string][]string)
header["Proxy-Authenticate"] = []string{"Basic"}
res := &http.Response{
Status: "407 Not authorized",
StatusCode: 407,
Proto: "HTTP/1.1",
ProtoMajor: 1,
ProtoMinor: 1,
Header: header,
}
return res
}

45
models/plugin/plugin.go Normal file
View File

@@ -0,0 +1,45 @@
// Copyright 2017 fatedier, fatedier@gmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package plugin
import (
"fmt"
"io"
)
// Creators is used for create plugins to handle connections.
var creators = make(map[string]CreatorFn)
// params has prefix "plugin_"
type CreatorFn func(params map[string]string) (Plugin, error)
func Register(name string, fn CreatorFn) {
creators[name] = fn
}
func Create(name string, params map[string]string) (p Plugin, err error) {
if fn, ok := creators[name]; ok {
p, err = fn(params)
} else {
err = fmt.Errorf("plugin [%s] is not registered", name)
}
return
}
type Plugin interface {
Name() string
Handle(conn io.ReadWriteCloser)
Close() error
}

65
models/plugin/socks5.go Normal file
View File

@@ -0,0 +1,65 @@
// Copyright 2017 fatedier, fatedier@gmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package plugin
import (
"io"
"io/ioutil"
"log"
frpNet "github.com/fatedier/frp/utils/net"
gosocks5 "github.com/armon/go-socks5"
)
const PluginSocks5 = "socks5"
func init() {
Register(PluginSocks5, NewSocks5Plugin)
}
type Socks5Plugin struct {
Server *gosocks5.Server
}
func NewSocks5Plugin(params map[string]string) (p Plugin, err error) {
sp := &Socks5Plugin{}
sp.Server, err = gosocks5.New(&gosocks5.Config{
Logger: log.New(ioutil.Discard, "", log.LstdFlags),
})
p = sp
return
}
func (sp *Socks5Plugin) Handle(conn io.ReadWriteCloser) {
defer conn.Close()
var wrapConn frpNet.Conn
if realConn, ok := conn.(frpNet.Conn); ok {
wrapConn = realConn
} else {
wrapConn = frpNet.WrapReadWriteCloserToConn(conn)
}
sp.Server.ServeConn(wrapConn)
}
func (sp *Socks5Plugin) Name() string {
return PluginSocks5
}
func (sp *Socks5Plugin) Close() error {
return nil
}

View File

@@ -0,0 +1,69 @@
// Copyright 2017 fatedier, fatedier@gmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package plugin
import (
"fmt"
"io"
"net"
frpIo "github.com/fatedier/frp/utils/io"
)
const PluginUnixDomainSocket = "unix_domain_socket"
func init() {
Register(PluginUnixDomainSocket, NewUnixDomainSocketPlugin)
}
type UnixDomainSocketPlugin struct {
UnixAddr *net.UnixAddr
}
func NewUnixDomainSocketPlugin(params map[string]string) (p Plugin, err error) {
unixPath, ok := params["plugin_unix_path"]
if !ok {
err = fmt.Errorf("plugin_unix_path not found")
return
}
unixAddr, errRet := net.ResolveUnixAddr("unix", unixPath)
if errRet != nil {
err = errRet
return
}
p = &UnixDomainSocketPlugin{
UnixAddr: unixAddr,
}
return
}
func (uds *UnixDomainSocketPlugin) Handle(conn io.ReadWriteCloser) {
localConn, err := net.DialUnix("unix", nil, uds.UnixAddr)
if err != nil {
return
}
frpIo.Join(localConn, conn)
}
func (uds *UnixDomainSocketPlugin) Name() string {
return PluginUnixDomainSocket
}
func (uds *UnixDomainSocketPlugin) Close() error {
return nil
}

View File

@@ -1,38 +0,0 @@
// Copyright 2016 fatedier, fatedier@gmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tcp
import (
"io"
"sync"
)
// Join two io.ReadWriteCloser and do some operations.
func Join(c1 io.ReadWriteCloser, c2 io.ReadWriteCloser) (inCount int64, outCount int64) {
var wait sync.WaitGroup
pipe := func(to io.ReadWriteCloser, from io.ReadWriteCloser, count *int64) {
defer to.Close()
defer from.Close()
defer wait.Done()
*count, _ = io.Copy(to, from)
}
wait.Add(2)
go pipe(c1, c2, &inCount)
go pipe(c2, c1, &outCount)
wait.Wait()
return
}

View File

@@ -1,67 +0,0 @@
// Copyright 2017 fatedier, fatedier@gmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tcp
import (
"io"
"testing"
"github.com/stretchr/testify/assert"
)
func TestJoin(t *testing.T) {
assert := assert.New(t)
var (
n int
err error
)
text1 := "A document that gives tips for writing clear, idiomatic Go code. A must read for any new Go programmer. It augments the tour and the language specification, both of which should be read first."
text2 := "A document that specifies the conditions under which reads of a variable in one goroutine can be guaranteed to observe values produced by writes to the same variable in a different goroutine."
// Forward bytes directly.
pr, pw := io.Pipe()
pr2, pw2 := io.Pipe()
pr3, pw3 := io.Pipe()
pr4, pw4 := io.Pipe()
conn1 := WrapReadWriteCloser(pr, pw2)
conn2 := WrapReadWriteCloser(pr2, pw)
conn3 := WrapReadWriteCloser(pr3, pw4)
conn4 := WrapReadWriteCloser(pr4, pw3)
go func() {
Join(conn2, conn3)
}()
buf1 := make([]byte, 1024)
buf2 := make([]byte, 1024)
conn1.Write([]byte(text1))
conn4.Write([]byte(text2))
n, err = conn4.Read(buf1)
assert.NoError(err)
assert.Equal(text1, string(buf1[:n]))
n, err = conn1.Read(buf2)
assert.NoError(err)
assert.Equal(text2, string(buf2[:n]))
conn1.Close()
conn2.Close()
conn3.Close()
conn4.Close()
}

View File

@@ -50,7 +50,7 @@ type Control struct {
workConnCh chan net.Conn
// proxies in one client
proxies []Proxy
proxies map[string]Proxy
// pool count
poolCount int
@@ -82,7 +82,7 @@ func NewControl(svr *Service, ctlConn net.Conn, loginMsg *msg.Login) *Control {
sendCh: make(chan msg.Message, 10),
readCh: make(chan msg.Message, 10),
workConnCh: make(chan net.Conn, loginMsg.PoolCount+10),
proxies: make([]Proxy, 0),
proxies: make(map[string]Proxy),
poolCount: loginMsg.PoolCount,
lastPing: time.Now(),
runId: loginMsg.RunId,
@@ -265,10 +265,12 @@ func (ctl *Control) stoper() {
workConn.Close()
}
ctl.mu.Lock()
defer ctl.mu.Unlock()
for _, pxy := range ctl.proxies {
pxy.Close()
ctl.svr.DelProxy(pxy.GetName())
StatsCloseProxy(pxy.GetConf().GetBaseInfo().ProxyType)
StatsCloseProxy(pxy.GetName(), pxy.GetConf().GetBaseInfo().ProxyType)
}
ctl.allShutdown.Done()
@@ -317,6 +319,9 @@ func (ctl *Control) manager() {
StatsNewProxy(m.ProxyName, m.ProxyType)
}
ctl.sendCh <- resp
case *msg.CloseProxy:
ctl.CloseProxy(m)
ctl.conn.Info("close proxy [%s] success", m.ProxyName)
case *msg.Ping:
ctl.lastPing = time.Now()
ctl.conn.Debug("receive heartbeat")
@@ -355,6 +360,25 @@ func (ctl *Control) RegisterProxy(pxyMsg *msg.NewProxy) (err error) {
if err != nil {
return err
}
ctl.proxies = append(ctl.proxies, pxy)
ctl.mu.Lock()
ctl.proxies[pxy.GetName()] = pxy
ctl.mu.Unlock()
return nil
}
func (ctl *Control) CloseProxy(closeMsg *msg.CloseProxy) (err error) {
ctl.mu.Lock()
defer ctl.mu.Unlock()
pxy, ok := ctl.proxies[closeMsg.ProxyName]
if !ok {
return
}
pxy.Close()
ctl.svr.DelProxy(pxy.GetName())
delete(ctl.proxies, closeMsg.ProxyName)
StatsCloseProxy(pxy.GetName(), pxy.GetConf().GetBaseInfo().ProxyType)
return
}

View File

@@ -22,6 +22,7 @@ import (
"github.com/fatedier/frp/assets"
"github.com/fatedier/frp/models/config"
frpNet "github.com/fatedier/frp/utils/net"
"github.com/julienschmidt/httprouter"
)
@@ -35,20 +36,24 @@ func RunDashboardServer(addr string, port int64) (err error) {
// url router
router := httprouter.New()
user, passwd := config.ServerCommonCfg.DashboardUser, config.ServerCommonCfg.DashboardPwd
// api, see dashboard_api.go
router.GET("/api/serverinfo", httprouterBasicAuth(apiServerInfo))
router.GET("/api/proxy/tcp", httprouterBasicAuth(apiProxyTcp))
router.GET("/api/proxy/udp", httprouterBasicAuth(apiProxyUdp))
router.GET("/api/proxy/http", httprouterBasicAuth(apiProxyHttp))
router.GET("/api/proxy/https", httprouterBasicAuth(apiProxyHttps))
router.GET("/api/proxy/traffic/:name", httprouterBasicAuth(apiProxyTraffic))
router.GET("/api/serverinfo", frpNet.HttprouterBasicAuth(apiServerInfo, user, passwd))
router.GET("/api/proxy/tcp", frpNet.HttprouterBasicAuth(apiProxyTcp, user, passwd))
router.GET("/api/proxy/udp", frpNet.HttprouterBasicAuth(apiProxyUdp, user, passwd))
router.GET("/api/proxy/http", frpNet.HttprouterBasicAuth(apiProxyHttp, user, passwd))
router.GET("/api/proxy/https", frpNet.HttprouterBasicAuth(apiProxyHttps, user, passwd))
router.GET("/api/proxy/traffic/:name", frpNet.HttprouterBasicAuth(apiProxyTraffic, user, passwd))
// view
router.Handler("GET", "/favicon.ico", http.FileServer(assets.FileSystem))
router.Handler("GET", "/static/*filepath", basicAuthWraper(http.StripPrefix("/static/", http.FileServer(assets.FileSystem))))
router.HandlerFunc("GET", "/", basicAuth(func(w http.ResponseWriter, r *http.Request) {
router.Handler("GET", "/static/*filepath", frpNet.MakeHttpGzipHandler(
frpNet.NewHttpBasicAuthWraper(http.StripPrefix("/static/", http.FileServer(assets.FileSystem)), user, passwd)))
router.HandlerFunc("GET", "/", frpNet.HttpBasicAuth(func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, "/static/", http.StatusMovedPermanently)
}))
}, user, passwd))
address := fmt.Sprintf("%s:%d", addr, port)
server := &http.Server{
@@ -68,60 +73,3 @@ func RunDashboardServer(addr string, port int64) (err error) {
go server.Serve(ln)
return
}
func use(h http.HandlerFunc, middleware ...func(http.HandlerFunc) http.HandlerFunc) http.HandlerFunc {
for _, m := range middleware {
h = m(h)
}
return h
}
type AuthWraper struct {
h http.Handler
user string
passwd string
}
func (aw *AuthWraper) ServeHTTP(w http.ResponseWriter, r *http.Request) {
user, passwd, hasAuth := r.BasicAuth()
if (aw.user == "" && aw.passwd == "") || (hasAuth && user == aw.user || passwd == aw.passwd) {
aw.h.ServeHTTP(w, r)
} else {
w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
}
}
func basicAuthWraper(h http.Handler) http.Handler {
return &AuthWraper{
h: h,
user: config.ServerCommonCfg.DashboardUser,
passwd: config.ServerCommonCfg.DashboardPwd,
}
}
func basicAuth(h http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
user, passwd, hasAuth := r.BasicAuth()
if (config.ServerCommonCfg.DashboardUser == "" && config.ServerCommonCfg.DashboardPwd == "") ||
(hasAuth && user == config.ServerCommonCfg.DashboardUser || passwd == config.ServerCommonCfg.DashboardPwd) {
h.ServeHTTP(w, r)
} else {
w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
}
}
}
func httprouterBasicAuth(h httprouter.Handle) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
user, passwd, hasAuth := r.BasicAuth()
if (config.ServerCommonCfg.DashboardUser == "" && config.ServerCommonCfg.DashboardPwd == "") ||
(hasAuth && user == config.ServerCommonCfg.DashboardUser || passwd == config.ServerCommonCfg.DashboardPwd) {
h(w, r, ps)
} else {
w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
}
}
}

View File

@@ -21,6 +21,7 @@ import (
"github.com/fatedier/frp/models/config"
"github.com/fatedier/frp/models/consts"
"github.com/fatedier/frp/utils/log"
"github.com/fatedier/frp/utils/version"
"github.com/julienschmidt/httprouter"
)
@@ -34,6 +35,7 @@ type GeneralResponse struct {
type ServerInfoResp struct {
GeneralResponse
Version string `json:"version"`
VhostHttpPort int64 `json:"vhost_http_port"`
VhostHttpsPort int64 `json:"vhost_https_port"`
AuthTimeout int64 `json:"auth_timeout"`
@@ -61,6 +63,7 @@ func apiServerInfo(w http.ResponseWriter, r *http.Request, _ httprouter.Params)
cfg := config.ServerCommonCfg
serverStats := StatsGetServer()
res = ServerInfoResp{
Version: version.Full(),
VhostHttpPort: cfg.VhostHttpPort,
VhostHttpsPort: cfg.VhostHttpsPort,
AuthTimeout: cfg.AuthTimeout,
@@ -86,6 +89,8 @@ type ProxyStatsInfo struct {
TodayTrafficIn int64 `json:"today_traffic_in"`
TodayTrafficOut int64 `json:"today_traffic_out"`
CurConns int64 `json:"cur_conns"`
LastStartTime string `json:"last_start_time"`
LastCloseTime string `json:"last_close_time"`
Status string `json:"status"`
}
@@ -173,10 +178,12 @@ func getProxyStatsByType(proxyType string) (proxyInfos []*ProxyStatsInfo) {
} else {
proxyInfo.Status = consts.Offline
}
proxyInfo.Name = ps.Name
proxyInfo.TodayTrafficIn = ps.TodayTrafficIn
proxyInfo.TodayTrafficOut = ps.TodayTrafficOut
proxyInfo.CurConns = ps.CurConns
proxyInfo.Name = ps.Name
proxyInfo.LastStartTime = ps.LastStartTime
proxyInfo.LastCloseTime = ps.LastCloseTime
proxyInfos = append(proxyInfos, proxyInfo)
}
return

View File

@@ -16,7 +16,12 @@ package server
import (
"fmt"
"io"
"sync"
frpIo "github.com/fatedier/frp/utils/io"
frpNet "github.com/fatedier/frp/utils/net"
"github.com/fatedier/frp/utils/util"
)
type ControlManager struct {
@@ -87,3 +92,72 @@ func (pm *ProxyManager) GetByName(name string) (pxy Proxy, ok bool) {
pxy, ok = pm.pxys[name]
return
}
// Manager for vistor listeners.
type VistorManager struct {
vistorListeners map[string]*frpNet.CustomListener
skMap map[string]string
mu sync.RWMutex
}
func NewVistorManager() *VistorManager {
return &VistorManager{
vistorListeners: make(map[string]*frpNet.CustomListener),
skMap: make(map[string]string),
}
}
func (vm *VistorManager) Listen(name string, sk string) (l *frpNet.CustomListener, err error) {
vm.mu.Lock()
defer vm.mu.Unlock()
if _, ok := vm.vistorListeners[name]; ok {
err = fmt.Errorf("custom listener for [%s] is repeated", name)
return
}
l = frpNet.NewCustomListener()
vm.vistorListeners[name] = l
vm.skMap[name] = sk
return
}
func (vm *VistorManager) NewConn(name string, conn frpNet.Conn, timestamp int64, signKey string,
useEncryption bool, useCompression bool) (err error) {
vm.mu.RLock()
defer vm.mu.RUnlock()
if l, ok := vm.vistorListeners[name]; ok {
var sk string
if sk = vm.skMap[name]; util.GetAuthKey(sk, timestamp) != signKey {
err = fmt.Errorf("vistor connection of [%s] auth failed", name)
return
}
var rwc io.ReadWriteCloser = conn
if useEncryption {
if rwc, err = frpIo.WithEncryption(rwc, []byte(sk)); err != nil {
err = fmt.Errorf("create encryption connection failed: %v", err)
return
}
}
if useCompression {
rwc = frpIo.WithCompression(rwc)
}
err = l.PutConn(frpNet.WrapReadWriteCloserToConn(rwc))
} else {
err = fmt.Errorf("custom listener for [%s] doesn't exist", name)
return
}
return
}
func (vm *VistorManager) CloseListener(name string) {
vm.mu.Lock()
defer vm.mu.Unlock()
delete(vm.vistorListeners, name)
delete(vm.skMap, name)
}

View File

@@ -16,8 +16,10 @@ package server
import (
"sync"
"time"
"github.com/fatedier/frp/models/config"
"github.com/fatedier/frp/utils/log"
"github.com/fatedier/frp/utils/metric"
)
@@ -46,10 +48,13 @@ type ServerStatistics struct {
}
type ProxyStatistics struct {
ProxyType string
TrafficIn metric.DateCounter
TrafficOut metric.DateCounter
CurConns metric.Counter
Name string
ProxyType string
TrafficIn metric.DateCounter
TrafficOut metric.DateCounter
CurConns metric.Counter
LastStartTime time.Time
LastCloseTime time.Time
}
func init() {
@@ -63,6 +68,27 @@ func init() {
ProxyStatistics: make(map[string]*ProxyStatistics),
}
go func() {
for {
time.Sleep(12 * time.Hour)
log.Debug("start to clear useless proxy statistics data...")
StatsClearUselessInfo()
log.Debug("finish to clear useless proxy statistics data")
}
}()
}
func StatsClearUselessInfo() {
// To check if there are proxies that closed than 7 days and drop them.
globalStats.mu.Lock()
defer globalStats.mu.Unlock()
for name, data := range globalStats.ProxyStatistics {
if !data.LastCloseTime.IsZero() && time.Since(data.LastCloseTime) > time.Duration(7*24)*time.Hour {
delete(globalStats.ProxyStatistics, name)
log.Trace("clear proxy [%s]'s statistics data, lastCloseTime: [%s]", name, data.LastCloseTime.String())
}
}
}
func StatsNewClient() {
@@ -91,6 +117,7 @@ func StatsNewProxy(name string, proxyType string) {
proxyStats, ok := globalStats.ProxyStatistics[name]
if !(ok && proxyStats.ProxyType == proxyType) {
proxyStats = &ProxyStatistics{
Name: name,
ProxyType: proxyType,
CurConns: metric.NewCounter(),
TrafficIn: metric.NewDateCounter(ReserveDays),
@@ -98,16 +125,20 @@ func StatsNewProxy(name string, proxyType string) {
}
globalStats.ProxyStatistics[name] = proxyStats
}
proxyStats.LastStartTime = time.Now()
}
}
func StatsCloseProxy(proxyType string) {
func StatsCloseProxy(proxyName string, proxyType string) {
if config.ServerCommonCfg.DashboardPort != 0 {
globalStats.mu.Lock()
defer globalStats.mu.Unlock()
if counter, ok := globalStats.ProxyTypeCounts[proxyType]; ok {
counter.Dec(1)
}
if proxyStats, ok := globalStats.ProxyStatistics[proxyName]; ok {
proxyStats.LastCloseTime = time.Now()
}
}
}
@@ -199,6 +230,8 @@ type ProxyStats struct {
Type string
TodayTrafficIn int64
TodayTrafficOut int64
LastStartTime string
LastCloseTime string
CurConns int64
}
@@ -219,6 +252,12 @@ func StatsGetProxiesByType(proxyType string) []*ProxyStats {
TodayTrafficOut: proxyStats.TrafficOut.TodayCount(),
CurConns: proxyStats.CurConns.Count(),
}
if !proxyStats.LastStartTime.IsZero() {
ps.LastStartTime = proxyStats.LastStartTime.Format("01-02 15:04:05")
}
if !proxyStats.LastCloseTime.IsZero() {
ps.LastCloseTime = proxyStats.LastCloseTime.Format("01-02 15:04:05")
}
res = append(res, ps)
}
return res

View File

@@ -24,9 +24,9 @@ import (
"github.com/fatedier/frp/models/config"
"github.com/fatedier/frp/models/msg"
"github.com/fatedier/frp/models/proto/tcp"
"github.com/fatedier/frp/models/proto/udp"
"github.com/fatedier/frp/utils/errors"
frpIo "github.com/fatedier/frp/utils/io"
"github.com/fatedier/frp/utils/log"
frpNet "github.com/fatedier/frp/utils/net"
"github.com/fatedier/frp/utils/vhost"
@@ -143,6 +143,11 @@ func NewProxy(ctl *Control, pxyConf config.ProxyConf) (pxy Proxy, err error) {
BaseProxy: basePxy,
cfg: cfg,
}
case *config.StcpProxyConf:
pxy = &StcpProxy{
BaseProxy: basePxy,
cfg: cfg,
}
default:
return pxy, fmt.Errorf("proxy type not support")
}
@@ -156,7 +161,7 @@ type TcpProxy struct {
}
func (pxy *TcpProxy) Run() error {
listener, err := frpNet.ListenTcp(config.ServerCommonCfg.BindAddr, pxy.cfg.RemotePort)
listener, err := frpNet.ListenTcp(config.ServerCommonCfg.ProxyBindAddr, pxy.cfg.RemotePort)
if err != nil {
return err
}
@@ -274,6 +279,33 @@ func (pxy *HttpsProxy) Close() {
pxy.BaseProxy.Close()
}
type StcpProxy struct {
BaseProxy
cfg *config.StcpProxyConf
}
func (pxy *StcpProxy) Run() error {
listener, err := pxy.ctl.svr.vistorManager.Listen(pxy.GetName(), pxy.cfg.Sk)
if err != nil {
return err
}
listener.AddLogPrefix(pxy.name)
pxy.listeners = append(pxy.listeners, listener)
pxy.Info("stcp proxy custom listen success")
pxy.startListenHandler(pxy, HandleUserTcpConnection)
return nil
}
func (pxy *StcpProxy) GetConf() config.ProxyConf {
return pxy.cfg
}
func (pxy *StcpProxy) Close() {
pxy.BaseProxy.Close()
pxy.ctl.svr.vistorManager.CloseListener(pxy.GetName())
}
type UdpProxy struct {
BaseProxy
cfg *config.UdpProxyConf
@@ -298,7 +330,7 @@ type UdpProxy struct {
}
func (pxy *UdpProxy) Run() (err error) {
addr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", config.ServerCommonCfg.BindAddr, pxy.cfg.RemotePort))
addr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", config.ServerCommonCfg.ProxyBindAddr, pxy.cfg.RemotePort))
if err != nil {
return err
}
@@ -461,20 +493,20 @@ func HandleUserTcpConnection(pxy Proxy, userConn frpNet.Conn) {
var local io.ReadWriteCloser = workConn
cfg := pxy.GetConf().GetBaseInfo()
if cfg.UseEncryption {
local, err = tcp.WithEncryption(local, []byte(config.ServerCommonCfg.PrivilegeToken))
local, err = frpIo.WithEncryption(local, []byte(config.ServerCommonCfg.PrivilegeToken))
if err != nil {
pxy.Error("create encryption stream error: %v", err)
return
}
}
if cfg.UseCompression {
local = tcp.WithCompression(local)
local = frpIo.WithCompression(local)
}
pxy.Debug("join connections, workConn(l[%s] r[%s]) userConn(l[%s] r[%s])", workConn.LocalAddr().String(),
workConn.RemoteAddr().String(), userConn.LocalAddr().String(), userConn.RemoteAddr().String())
StatsOpenConnection(pxy.GetName())
inCount, outCount := tcp.Join(local, userConn)
inCount, outCount := frpIo.Join(local, userConn)
StatsCloseConnection(pxy.GetName())
StatsAddTrafficIn(pxy.GetName(), inCount)
StatsAddTrafficOut(pxy.GetName(), outCount)

View File

@@ -41,6 +41,9 @@ type Service struct {
// Accept connections from client.
listener frpNet.Listener
// Accept connections using kcp.
kcpListener frpNet.Listener
// For http proxies, route requests to different clients by hostname and other infomation.
VhostHttpMuxer *vhost.HttpMuxer
@@ -52,12 +55,16 @@ type Service struct {
// Manage all proxies.
pxyManager *ProxyManager
// Manage all vistor listeners.
vistorManager *VistorManager
}
func NewService() (svr *Service, err error) {
svr = &Service{
ctlManager: NewControlManager(),
pxyManager: NewProxyManager(),
ctlManager: NewControlManager(),
pxyManager: NewProxyManager(),
vistorManager: NewVistorManager(),
}
// Init assets.
@@ -73,11 +80,22 @@ func NewService() (svr *Service, err error) {
err = fmt.Errorf("Create server listener error, %v", err)
return
}
log.Info("frps tcp listen on %s:%d", config.ServerCommonCfg.BindAddr, config.ServerCommonCfg.BindPort)
// Listen for accepting connections from client using kcp protocol.
if config.ServerCommonCfg.KcpBindPort > 0 {
svr.kcpListener, err = frpNet.ListenKcp(config.ServerCommonCfg.BindAddr, config.ServerCommonCfg.KcpBindPort)
if err != nil {
err = fmt.Errorf("Listen on kcp address udp [%s:%d] error: %v", config.ServerCommonCfg.BindAddr, config.ServerCommonCfg.KcpBindPort, err)
return
}
log.Info("frps kcp listen on udp %s:%d", config.ServerCommonCfg.BindAddr, config.ServerCommonCfg.BindPort)
}
// Create http vhost muxer.
if config.ServerCommonCfg.VhostHttpPort != 0 {
if config.ServerCommonCfg.VhostHttpPort > 0 {
var l frpNet.Listener
l, err = frpNet.ListenTcp(config.ServerCommonCfg.BindAddr, config.ServerCommonCfg.VhostHttpPort)
l, err = frpNet.ListenTcp(config.ServerCommonCfg.ProxyBindAddr, config.ServerCommonCfg.VhostHttpPort)
if err != nil {
err = fmt.Errorf("Create vhost http listener error, %v", err)
return
@@ -87,12 +105,13 @@ func NewService() (svr *Service, err error) {
err = fmt.Errorf("Create vhost httpMuxer error, %v", err)
return
}
log.Info("http service listen on %s:%d", config.ServerCommonCfg.ProxyBindAddr, config.ServerCommonCfg.VhostHttpPort)
}
// Create https vhost muxer.
if config.ServerCommonCfg.VhostHttpsPort != 0 {
if config.ServerCommonCfg.VhostHttpsPort > 0 {
var l frpNet.Listener
l, err = frpNet.ListenTcp(config.ServerCommonCfg.BindAddr, config.ServerCommonCfg.VhostHttpsPort)
l, err = frpNet.ListenTcp(config.ServerCommonCfg.ProxyBindAddr, config.ServerCommonCfg.VhostHttpsPort)
if err != nil {
err = fmt.Errorf("Create vhost https listener error, %v", err)
return
@@ -102,10 +121,11 @@ func NewService() (svr *Service, err error) {
err = fmt.Errorf("Create vhost httpsMuxer error, %v", err)
return
}
log.Info("https service listen on %s:%d", config.ServerCommonCfg.ProxyBindAddr, config.ServerCommonCfg.VhostHttpsPort)
}
// Create dashboard web server.
if config.ServerCommonCfg.DashboardPort != 0 {
if config.ServerCommonCfg.DashboardPort > 0 {
err = RunDashboardServer(config.ServerCommonCfg.BindAddr, config.ServerCommonCfg.DashboardPort)
if err != nil {
err = fmt.Errorf("Create dashboard web server error, %v", err)
@@ -117,9 +137,17 @@ func NewService() (svr *Service, err error) {
}
func (svr *Service) Run() {
if config.ServerCommonCfg.KcpBindPort > 0 {
go svr.HandleListener(svr.kcpListener)
}
svr.HandleListener(svr.listener)
}
func (svr *Service) HandleListener(l frpNet.Listener) {
// Listen for incoming connections from client.
for {
c, err := svr.listener.Accept()
c, err := l.Accept()
if err != nil {
log.Warn("Listener for incoming connections from client closed")
return
@@ -131,7 +159,7 @@ func (svr *Service) Run() {
var rawMsg msg.Message
conn.SetReadDeadline(time.Now().Add(connReadTimeout))
if rawMsg, err = msg.ReadMsg(conn); err != nil {
log.Warn("Failed to read message: %v", err)
log.Trace("Failed to read message: %v", err)
conn.Close()
return
}
@@ -152,6 +180,20 @@ func (svr *Service) Run() {
}
case *msg.NewWorkConn:
svr.RegisterWorkConn(conn, m)
case *msg.NewVistorConn:
if err = svr.RegisterVistorConn(conn, m); err != nil {
conn.Warn("%v", err)
msg.WriteMsg(conn, &msg.NewVistorConnResp{
ProxyName: m.ProxyName,
Error: err.Error(),
})
conn.Close()
} else {
msg.WriteMsg(conn, &msg.NewVistorConnResp{
ProxyName: m.ProxyName,
Error: "",
})
}
default:
log.Warn("Error message type for the new connection [%s]", conn.RemoteAddr().String())
conn.Close()
@@ -238,9 +280,13 @@ func (svr *Service) RegisterWorkConn(workConn frpNet.Conn, newMsg *msg.NewWorkCo
return
}
func (svr *Service) RegisterVistorConn(vistorConn frpNet.Conn, newMsg *msg.NewVistorConn) error {
return svr.vistorManager.NewConn(newMsg.ProxyName, vistorConn, newMsg.Timestamp, newMsg.SignKey,
newMsg.UseEncryption, newMsg.UseCompression)
}
func (svr *Service) RegisterProxy(name string, pxy Proxy) error {
err := svr.pxyManager.Add(name, pxy)
return err
return svr.pxyManager.Add(name, pxy)
}
func (svr *Service) DelProxy(name string) {

View File

@@ -27,3 +27,9 @@ type = udp
local_ip = 127.0.0.1
local_port = 10703
remote_port = 10712
[unix_domain]
type = tcp
remote_port = 10704
plugin = unix_domain_socket
plugin_unix_path = /tmp/frp_echo_server.sock

View File

@@ -4,12 +4,15 @@ import (
"bufio"
"fmt"
"io"
"net"
"os"
"syscall"
"github.com/fatedier/frp/utils/net"
frpNet "github.com/fatedier/frp/utils/net"
)
func StartEchoServer() {
l, err := net.ListenTcp("127.0.0.1", 10701)
l, err := frpNet.ListenTcp("127.0.0.1", 10701)
if err != nil {
fmt.Printf("echo server listen error: %v\n", err)
return
@@ -27,7 +30,7 @@ func StartEchoServer() {
}
func StartUdpEchoServer() {
l, err := net.ListenUDP("127.0.0.1", 10703)
l, err := frpNet.ListenUDP("127.0.0.1", 10703)
if err != nil {
fmt.Printf("udp echo server listen error: %v\n", err)
return
@@ -44,6 +47,27 @@ func StartUdpEchoServer() {
}
}
func StartUnixDomainServer() {
unixPath := "/tmp/frp_echo_server.sock"
os.Remove(unixPath)
syscall.Umask(0)
l, err := net.Listen("unix", unixPath)
if err != nil {
fmt.Printf("unix domain server listen error: %v\n", err)
return
}
for {
c, err := l.Accept()
if err != nil {
fmt.Printf("unix domain server accept error: %v\n", err)
return
}
go echoWorker(c)
}
}
func echoWorker(c net.Conn) {
br := bufio.NewReader(c)
for {

View File

@@ -26,6 +26,7 @@ func init() {
go StartEchoServer()
go StartUdpEchoServer()
go StartHttpServer()
go StartUnixDomainServer()
time.Sleep(500 * time.Millisecond)
}
@@ -95,3 +96,24 @@ func TestUdpEchoServer(t *testing.T) {
t.Fatalf("message got from udp server error, get %s", string(data[:n-1]))
}
}
func TestUnixDomainServer(t *testing.T) {
c, err := frpNet.ConnectTcpServer(fmt.Sprintf("127.0.0.1:%d", 10704))
if err != nil {
t.Fatalf("connect to echo server error: %v", err)
}
timer := time.Now().Add(time.Duration(5) * time.Second)
c.SetDeadline(timer)
c.Write([]byte(ECHO_TEST_STR + "\n"))
br := bufio.NewReader(c)
buf, err := br.ReadString('\n')
if err != nil {
t.Fatalf("read from echo server error: %v", err)
}
if ECHO_TEST_STR != buf {
t.Fatalf("content error, send [%s], get [%s]", strings.Trim(ECHO_TEST_STR, "\n"), strings.Trim(buf, "\n"))
}
}

View File

@@ -12,38 +12,74 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package tcp
package io
import (
"io"
"github.com/golang/snappy"
"sync"
"github.com/fatedier/frp/utils/crypto"
"github.com/fatedier/frp/utils/pool"
)
// Join two io.ReadWriteCloser and do some operations.
func Join(c1 io.ReadWriteCloser, c2 io.ReadWriteCloser) (inCount int64, outCount int64) {
var wait sync.WaitGroup
pipe := func(to io.ReadWriteCloser, from io.ReadWriteCloser, count *int64) {
defer to.Close()
defer from.Close()
defer wait.Done()
buf := pool.GetBuf(16 * 1024)
defer pool.PutBuf(buf)
*count, _ = io.CopyBuffer(to, from, buf)
}
wait.Add(2)
go pipe(c1, c2, &inCount)
go pipe(c2, c1, &outCount)
wait.Wait()
return
}
func WithEncryption(rwc io.ReadWriteCloser, key []byte) (io.ReadWriteCloser, error) {
w, err := crypto.NewWriter(rwc, key)
if err != nil {
return nil, err
}
return WrapReadWriteCloser(crypto.NewReader(rwc, key), w), nil
return WrapReadWriteCloser(crypto.NewReader(rwc, key), w, func() error {
return rwc.Close()
}), nil
}
func WithCompression(rwc io.ReadWriteCloser) io.ReadWriteCloser {
return WrapReadWriteCloser(snappy.NewReader(rwc), snappy.NewWriter(rwc))
}
func WrapReadWriteCloser(r io.Reader, w io.Writer) io.ReadWriteCloser {
return &ReadWriteCloser{
r: r,
w: w,
}
sr := pool.GetSnappyReader(rwc)
sw := pool.GetSnappyWriter(rwc)
return WrapReadWriteCloser(sr, sw, func() error {
err := rwc.Close()
pool.PutSnappyReader(sr)
pool.PutSnappyWriter(sw)
return err
})
}
type ReadWriteCloser struct {
r io.Reader
w io.Writer
r io.Reader
w io.Writer
closeFn func() error
closed bool
mu sync.Mutex
}
// closeFn will be called only once
func WrapReadWriteCloser(r io.Reader, w io.Writer, closeFn func() error) io.ReadWriteCloser {
return &ReadWriteCloser{
r: r,
w: w,
closeFn: closeFn,
closed: false,
}
}
func (rwc *ReadWriteCloser) Read(p []byte) (n int, err error) {
@@ -55,6 +91,14 @@ func (rwc *ReadWriteCloser) Write(p []byte) (n int, err error) {
}
func (rwc *ReadWriteCloser) Close() (errRet error) {
rwc.mu.Lock()
if rwc.closed {
rwc.mu.Unlock()
return
}
rwc.closed = true
rwc.mu.Unlock()
var err error
if rc, ok := rwc.r.(io.Closer); ok {
err = rc.Close()
@@ -69,5 +113,12 @@ func (rwc *ReadWriteCloser) Close() (errRet error) {
errRet = err
}
}
if rwc.closeFn != nil {
err = rwc.closeFn()
if err != nil {
errRet = err
}
}
return
}

View File

@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package tcp
package io
import (
"io"
@@ -21,6 +21,51 @@ import (
"github.com/stretchr/testify/assert"
)
func TestJoin(t *testing.T) {
assert := assert.New(t)
var (
n int
err error
)
text1 := "A document that gives tips for writing clear, idiomatic Go code. A must read for any new Go programmer. It augments the tour and the language specification, both of which should be read first."
text2 := "A document that specifies the conditions under which reads of a variable in one goroutine can be guaranteed to observe values produced by writes to the same variable in a different goroutine."
// Forward bytes directly.
pr, pw := io.Pipe()
pr2, pw2 := io.Pipe()
pr3, pw3 := io.Pipe()
pr4, pw4 := io.Pipe()
conn1 := WrapReadWriteCloser(pr, pw2, nil)
conn2 := WrapReadWriteCloser(pr2, pw, nil)
conn3 := WrapReadWriteCloser(pr3, pw4, nil)
conn4 := WrapReadWriteCloser(pr4, pw3, nil)
go func() {
Join(conn2, conn3)
}()
buf1 := make([]byte, 1024)
buf2 := make([]byte, 1024)
conn1.Write([]byte(text1))
conn4.Write([]byte(text2))
n, err = conn4.Read(buf1)
assert.NoError(err)
assert.Equal(text1, string(buf1[:n]))
n, err = conn1.Read(buf2)
assert.NoError(err)
assert.Equal(text2, string(buf2[:n]))
conn1.Close()
conn2.Close()
conn3.Close()
conn4.Close()
}
func TestWithCompression(t *testing.T) {
assert := assert.New(t)
@@ -28,8 +73,8 @@ func TestWithCompression(t *testing.T) {
pr, pw := io.Pipe()
pr2, pw2 := io.Pipe()
conn1 := WrapReadWriteCloser(pr, pw2)
conn2 := WrapReadWriteCloser(pr2, pw)
conn1 := WrapReadWriteCloser(pr, pw2, nil)
conn2 := WrapReadWriteCloser(pr2, pw, nil)
compressionStream1 := WithCompression(conn1)
compressionStream2 := WithCompression(conn2)
@@ -71,12 +116,12 @@ func TestWithEncryption(t *testing.T) {
pr5, pw5 := io.Pipe()
pr6, pw6 := io.Pipe()
conn1 := WrapReadWriteCloser(pr, pw2)
conn2 := WrapReadWriteCloser(pr2, pw)
conn3 := WrapReadWriteCloser(pr3, pw4)
conn4 := WrapReadWriteCloser(pr4, pw3)
conn5 := WrapReadWriteCloser(pr5, pw6)
conn6 := WrapReadWriteCloser(pr6, pw5)
conn1 := WrapReadWriteCloser(pr, pw2, nil)
conn2 := WrapReadWriteCloser(pr2, pw, nil)
conn3 := WrapReadWriteCloser(pr3, pw4, nil)
conn4 := WrapReadWriteCloser(pr4, pw3, nil)
conn5 := WrapReadWriteCloser(pr5, pw6, nil)
conn6 := WrapReadWriteCloser(pr6, pw5, nil)
encryptStream1, err := WithEncryption(conn3, []byte(key))
assert.NoError(err)

View File

@@ -88,6 +88,7 @@ func Trace(format string, v ...interface{}) {
// Logger
type Logger interface {
AddLogPrefix(string)
GetPrefixStr() string
GetAllPrefix() []string
ClearLogPrefix()
Error(string, ...interface{})
@@ -119,6 +120,10 @@ func (pl *PrefixLogger) AddLogPrefix(prefix string) {
pl.allPrefix = append(pl.allPrefix, prefix)
}
func (pl *PrefixLogger) GetPrefixStr() string {
return pl.prefix
}
func (pl *PrefixLogger) GetAllPrefix() []string {
return pl.allPrefix
}

View File

@@ -15,9 +15,17 @@
package net
import (
"bytes"
"errors"
"fmt"
"io"
"net"
"sync"
"time"
"github.com/fatedier/frp/utils/log"
kcp "github.com/xtaci/kcp-go"
)
// Conn is the interface of connections used in frp.
@@ -32,14 +40,119 @@ type WrapLogConn struct {
}
func WrapConn(c net.Conn) Conn {
return WrapLogConn{
return &WrapLogConn{
Conn: c,
Logger: log.NewPrefixLogger(""),
}
}
type Listener interface {
Accept() (Conn, error)
Close() error
type WrapReadWriteCloserConn struct {
io.ReadWriteCloser
log.Logger
}
func WrapReadWriteCloserToConn(rwc io.ReadWriteCloser) Conn {
return &WrapReadWriteCloserConn{
ReadWriteCloser: rwc,
Logger: log.NewPrefixLogger(""),
}
}
func (conn *WrapReadWriteCloserConn) LocalAddr() net.Addr {
return (*net.TCPAddr)(nil)
}
func (conn *WrapReadWriteCloserConn) RemoteAddr() net.Addr {
return (*net.TCPAddr)(nil)
}
func (conn *WrapReadWriteCloserConn) SetDeadline(t time.Time) error {
return &net.OpError{Op: "set", Net: "wrap", Source: nil, Addr: nil, Err: errors.New("deadline not supported")}
}
func (conn *WrapReadWriteCloserConn) SetReadDeadline(t time.Time) error {
return &net.OpError{Op: "set", Net: "wrap", Source: nil, Addr: nil, Err: errors.New("deadline not supported")}
}
func (conn *WrapReadWriteCloserConn) SetWriteDeadline(t time.Time) error {
return &net.OpError{Op: "set", Net: "wrap", Source: nil, Addr: nil, Err: errors.New("deadline not supported")}
}
func ConnectServer(protocol string, addr string) (c Conn, err error) {
switch protocol {
case "tcp":
return ConnectTcpServer(addr)
case "kcp":
kcpConn, errRet := kcp.DialWithOptions(addr, nil, 10, 3)
if errRet != nil {
err = errRet
return
}
kcpConn.SetStreamMode(true)
kcpConn.SetWriteDelay(true)
kcpConn.SetNoDelay(1, 20, 2, 1)
kcpConn.SetWindowSize(128, 512)
kcpConn.SetMtu(1350)
kcpConn.SetACKNoDelay(false)
kcpConn.SetReadBuffer(4194304)
kcpConn.SetWriteBuffer(4194304)
c = WrapConn(kcpConn)
return
default:
return nil, fmt.Errorf("unsupport protocol: %s", protocol)
}
}
func ConnectServerByHttpProxy(httpProxy string, protocol string, addr string) (c Conn, err error) {
switch protocol {
case "tcp":
return ConnectTcpServerByHttpProxy(httpProxy, addr)
case "kcp":
// http proxy is not supported for kcp
return ConnectServer(protocol, addr)
default:
return nil, fmt.Errorf("unsupport protocol: %s", protocol)
}
}
type SharedConn struct {
Conn
sync.Mutex
buf *bytes.Buffer
}
// the bytes you read in io.Reader, will be reserved in SharedConn
func NewShareConn(conn Conn) (*SharedConn, io.Reader) {
sc := &SharedConn{
Conn: conn,
buf: bytes.NewBuffer(make([]byte, 0, 1024)),
}
return sc, io.TeeReader(conn, sc.buf)
}
func (sc *SharedConn) Read(p []byte) (n int, err error) {
sc.Lock()
if sc.buf == nil {
sc.Unlock()
return sc.Conn.Read(p)
}
sc.Unlock()
n, err = sc.buf.Read(p)
if err == io.EOF {
sc.Lock()
sc.buf = nil
sc.Unlock()
var n2 int
n2, err = sc.Conn.Read(p[n:])
n += n2
}
return
}
func (sc *SharedConn) WriteBuff(buffer []byte) (err error) {
sc.buf.Reset()
_, err = sc.buf.Write(buffer)
return err
}

105
utils/net/http.go Normal file
View File

@@ -0,0 +1,105 @@
// Copyright 2017 fatedier, fatedier@gmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package net
import (
"compress/gzip"
"io"
"net/http"
"strings"
"github.com/julienschmidt/httprouter"
)
type HttpAuthWraper struct {
h http.Handler
user string
passwd string
}
func NewHttpBasicAuthWraper(h http.Handler, user, passwd string) http.Handler {
return &HttpAuthWraper{
h: h,
user: user,
passwd: passwd,
}
}
func (aw *HttpAuthWraper) ServeHTTP(w http.ResponseWriter, r *http.Request) {
user, passwd, hasAuth := r.BasicAuth()
if (aw.user == "" && aw.passwd == "") || (hasAuth && user == aw.user && passwd == aw.passwd) {
aw.h.ServeHTTP(w, r)
} else {
w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
}
}
func HttpBasicAuth(h http.HandlerFunc, user, passwd string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
reqUser, reqPasswd, hasAuth := r.BasicAuth()
if (user == "" && passwd == "") ||
(hasAuth && reqUser == user && reqPasswd == passwd) {
h.ServeHTTP(w, r)
} else {
w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
}
}
}
func HttprouterBasicAuth(h httprouter.Handle, user, passwd string) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
reqUser, reqPasswd, hasAuth := r.BasicAuth()
if (user == "" && passwd == "") ||
(hasAuth && reqUser == user && reqPasswd == passwd) {
h(w, r, ps)
} else {
w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
}
}
}
type HttpGzipWraper struct {
h http.Handler
}
func (gw *HttpGzipWraper) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
gw.h.ServeHTTP(w, r)
return
}
w.Header().Set("Content-Encoding", "gzip")
gz := gzip.NewWriter(w)
defer gz.Close()
gzr := gzipResponseWriter{Writer: gz, ResponseWriter: w}
gw.h.ServeHTTP(gzr, r)
}
func MakeHttpGzipHandler(h http.Handler) http.Handler {
return &HttpGzipWraper{
h: h,
}
}
type gzipResponseWriter struct {
io.Writer
http.ResponseWriter
}
func (w gzipResponseWriter) Write(b []byte) (int, error) {
return w.Writer.Write(b)
}

87
utils/net/kcp.go Normal file
View File

@@ -0,0 +1,87 @@
// Copyright 2017 fatedier, fatedier@gmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package net
import (
"fmt"
"net"
"github.com/fatedier/frp/utils/log"
kcp "github.com/xtaci/kcp-go"
)
type KcpListener struct {
net.Addr
listener net.Listener
accept chan Conn
closeFlag bool
log.Logger
}
func ListenKcp(bindAddr string, bindPort int64) (l *KcpListener, err error) {
listener, err := kcp.ListenWithOptions(fmt.Sprintf("%s:%d", bindAddr, bindPort), nil, 10, 3)
if err != nil {
return l, err
}
listener.SetReadBuffer(4194304)
listener.SetWriteBuffer(4194304)
l = &KcpListener{
Addr: listener.Addr(),
listener: listener,
accept: make(chan Conn),
closeFlag: false,
Logger: log.NewPrefixLogger(""),
}
go func() {
for {
conn, err := listener.AcceptKCP()
if err != nil {
if l.closeFlag {
close(l.accept)
return
}
continue
}
conn.SetStreamMode(true)
conn.SetWriteDelay(true)
conn.SetNoDelay(1, 20, 2, 1)
conn.SetMtu(1350)
conn.SetWindowSize(1024, 1024)
conn.SetACKNoDelay(false)
l.accept <- WrapConn(conn)
}
}()
return l, err
}
func (l *KcpListener) Accept() (Conn, error) {
conn, ok := <-l.accept
if !ok {
return conn, fmt.Errorf("channel for kcp listener closed")
}
return conn, nil
}
func (l *KcpListener) Close() error {
if !l.closeFlag {
l.closeFlag = true
l.listener.Close()
}
return nil
}

99
utils/net/listener.go Normal file
View File

@@ -0,0 +1,99 @@
// Copyright 2017 fatedier, fatedier@gmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package net
import (
"fmt"
"net"
"sync"
"github.com/fatedier/frp/utils/errors"
"github.com/fatedier/frp/utils/log"
)
type Listener interface {
Accept() (Conn, error)
Close() error
log.Logger
}
type LogListener struct {
l net.Listener
net.Listener
log.Logger
}
func WrapLogListener(l net.Listener) Listener {
return &LogListener{
l: l,
Listener: l,
Logger: log.NewPrefixLogger(""),
}
}
func (logL *LogListener) Accept() (Conn, error) {
c, err := logL.l.Accept()
return WrapConn(c), err
}
// Custom listener
type CustomListener struct {
conns chan Conn
closed bool
mu sync.Mutex
log.Logger
}
func NewCustomListener() *CustomListener {
return &CustomListener{
conns: make(chan Conn, 64),
Logger: log.NewPrefixLogger(""),
}
}
func (l *CustomListener) Accept() (Conn, error) {
conn, ok := <-l.conns
if !ok {
return nil, fmt.Errorf("listener closed")
}
conn.AddLogPrefix(l.GetPrefixStr())
return conn, nil
}
func (l *CustomListener) PutConn(conn Conn) error {
err := errors.PanicToError(func() {
select {
case l.conns <- conn:
default:
conn.Close()
}
})
return err
}
func (l *CustomListener) Close() error {
l.mu.Lock()
defer l.mu.Unlock()
if !l.closed {
close(l.conns)
l.closed = true
}
return nil
}
func (l *CustomListener) Addr() net.Addr {
return (*net.TCPAddr)(nil)
}

View File

@@ -128,7 +128,9 @@ func ConnectTcpServerByHttpProxy(httpProxy string, serverAddr string) (c Conn, e
var proxyAuth string
if proxyUrl.User != nil {
proxyAuth = "Basic " + base64.StdEncoding.EncodeToString([]byte(proxyUrl.User.String()))
username := proxyUrl.User.Username()
passwd, _ := proxyUrl.User.Password()
proxyAuth = "Basic " + base64.StdEncoding.EncodeToString([]byte(username+":"+passwd))
}
if proxyUrl.Scheme != "http" {

View File

@@ -17,15 +17,18 @@ package pool
import "sync"
var (
bufPool5k sync.Pool
bufPool2k sync.Pool
bufPool1k sync.Pool
bufPool sync.Pool
bufPool16k sync.Pool
bufPool5k sync.Pool
bufPool2k sync.Pool
bufPool1k sync.Pool
bufPool sync.Pool
)
func GetBuf(size int) []byte {
var x interface{}
if size >= 5*1024 {
if size >= 16*1024 {
x = bufPool16k.Get()
} else if size >= 5*1024 {
x = bufPool5k.Get()
} else if size >= 2*1024 {
x = bufPool2k.Get()
@@ -46,7 +49,9 @@ func GetBuf(size int) []byte {
func PutBuf(buf []byte) {
size := cap(buf)
if size >= 5*1024 {
if size >= 16*1024 {
bufPool16k.Put(buf)
} else if size >= 5*1024 {
bufPool5k.Put(buf)
} else if size >= 2*1024 {
bufPool2k.Put(buf)

57
utils/pool/snappy.go Normal file
View File

@@ -0,0 +1,57 @@
// Copyright 2017 fatedier, fatedier@gmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pool
import (
"io"
"sync"
"github.com/golang/snappy"
)
var (
snappyReaderPool sync.Pool
snappyWriterPool sync.Pool
)
func GetSnappyReader(r io.Reader) *snappy.Reader {
var x interface{}
x = snappyReaderPool.Get()
if x == nil {
return snappy.NewReader(r)
}
sr := x.(*snappy.Reader)
sr.Reset(r)
return sr
}
func PutSnappyReader(sr *snappy.Reader) {
snappyReaderPool.Put(sr)
}
func GetSnappyWriter(w io.Writer) *snappy.Writer {
var x interface{}
x = snappyWriterPool.Get()
if x == nil {
return snappy.NewWriter(w)
}
sw := x.(*snappy.Writer)
sw.Reset(w)
return sw
}
func PutSnappyWriter(sw *snappy.Writer) {
snappyWriterPool.Put(sw)
}

View File

@@ -19,7 +19,7 @@ import (
"strings"
)
var version string = "0.10.0"
var version string = "0.13.0"
func Full() string {
return version
@@ -54,8 +54,8 @@ func Minor(v string) int64 {
// add every case there if server will not accept client's protocol and return false
func Compat(client string) (ok bool, msg string) {
if LessThan(client, version) {
return false, "Please upgrade your frpc version to 0.10.0"
if LessThan(client, "0.10.0") {
return false, "Please upgrade your frpc version to at least 0.10.0"
}
return true, ""
}

View File

@@ -35,7 +35,7 @@ type HttpMuxer struct {
func GetHttpRequestInfo(c frpNet.Conn) (_ frpNet.Conn, _ map[string]string, err error) {
reqInfoMap := make(map[string]string, 0)
sc, rd := newShareConn(c)
sc, rd := frpNet.NewShareConn(c)
request, err := http.ReadRequest(bufio.NewReader(rd))
if err != nil {
@@ -57,30 +57,35 @@ func GetHttpRequestInfo(c frpNet.Conn) (_ frpNet.Conn, _ map[string]string, err
}
func NewHttpMuxer(listener frpNet.Listener, timeout time.Duration) (*HttpMuxer, error) {
mux, err := NewVhostMuxer(listener, GetHttpRequestInfo, HttpAuthFunc, HttpHostNameRewrite, timeout)
mux, err := NewVhostMuxer(listener, GetHttpRequestInfo, HttpAuthFunc, ModifyHttpRequest, timeout)
return &HttpMuxer{mux}, err
}
func HttpHostNameRewrite(c frpNet.Conn, rewriteHost string) (_ frpNet.Conn, err error) {
sc, rd := newShareConn(c)
func ModifyHttpRequest(c frpNet.Conn, rewriteHost string) (_ frpNet.Conn, err error) {
sc, rd := frpNet.NewShareConn(c)
var buff []byte
if buff, err = hostNameRewrite(rd, rewriteHost); err != nil {
remoteIP := strings.Split(c.RemoteAddr().String(), ":")[0]
if buff, err = hostNameRewrite(rd, rewriteHost, remoteIP); err != nil {
return sc, err
}
err = sc.WriteBuff(buff)
return sc, err
}
func hostNameRewrite(request io.Reader, rewriteHost string) (_ []byte, err error) {
func hostNameRewrite(request io.Reader, rewriteHost string, remoteIP string) (_ []byte, err error) {
buf := pool.GetBuf(1024)
defer pool.PutBuf(buf)
request.Read(buf)
retBuffer, err := parseRequest(buf, rewriteHost)
var n int
n, err = request.Read(buf)
if err != nil {
return
}
retBuffer, err := parseRequest(buf[:n], rewriteHost, remoteIP)
return retBuffer, err
}
func parseRequest(org []byte, rewriteHost string) (ret []byte, err error) {
func parseRequest(org []byte, rewriteHost string, remoteIP string) (ret []byte, err error) {
tp := bytes.NewBuffer(org)
// First line: GET /index.html HTTP/1.0
var b []byte
@@ -106,10 +111,19 @@ func parseRequest(org []byte, rewriteHost string) (ret []byte, err error) {
// GET /index.html HTTP/1.1
// Host: www.google.com
if req.URL.Host == "" {
changedBuf, err := changeHostName(tp, rewriteHost)
var changedBuf []byte
if rewriteHost != "" {
changedBuf, err = changeHostName(tp, rewriteHost)
}
buf := new(bytes.Buffer)
buf.Write(b)
buf.Write(changedBuf)
buf.WriteString(fmt.Sprintf("X-Forwarded-For: %s\r\n", remoteIP))
buf.WriteString(fmt.Sprintf("X-Real-IP: %s\r\n", remoteIP))
if len(changedBuf) == 0 {
tp.WriteTo(buf)
} else {
buf.Write(changedBuf)
}
return buf.Bytes(), err
}
@@ -117,18 +131,21 @@ func parseRequest(org []byte, rewriteHost string) (ret []byte, err error) {
// GET http://www.google.com/index.html HTTP/1.1
// Host: doesntmatter
// In this case, any Host line is ignored.
hostPort := strings.Split(req.URL.Host, ":")
if len(hostPort) == 1 {
req.URL.Host = rewriteHost
} else if len(hostPort) == 2 {
req.URL.Host = fmt.Sprintf("%s:%s", rewriteHost, hostPort[1])
if rewriteHost != "" {
hostPort := strings.Split(req.URL.Host, ":")
if len(hostPort) == 1 {
req.URL.Host = rewriteHost
} else if len(hostPort) == 2 {
req.URL.Host = fmt.Sprintf("%s:%s", rewriteHost, hostPort[1])
}
}
firstLine := req.Method + " " + req.URL.String() + " " + req.Proto
buf := new(bytes.Buffer)
buf.WriteString(firstLine)
buf.WriteString(fmt.Sprintf("X-Forwarded-For: %s\r\n", remoteIP))
buf.WriteString(fmt.Sprintf("X-Real-IP: %s\r\n", remoteIP))
tp.WriteTo(buf)
return buf.Bytes(), err
}
// parseRequestLine parses "GET /foo HTTP/1.1" into its three parts.
@@ -162,9 +179,9 @@ func changeHostName(buff *bytes.Buffer, rewriteHost string) (_ []byte, err error
var hostHeader string
portPos := bytes.IndexByte(kv[j+1:], ':')
if portPos == -1 {
hostHeader = fmt.Sprintf("Host: %s\n", rewriteHost)
hostHeader = fmt.Sprintf("Host: %s\r\n", rewriteHost)
} else {
hostHeader = fmt.Sprintf("Host: %s:%s\n", rewriteHost, kv[portPos+1:])
hostHeader = fmt.Sprintf("Host: %s:%s\r\n", rewriteHost, kv[j+portPos+2:])
}
retBuf.WriteString(hostHeader)
peek = peek[i+1:]

View File

@@ -179,7 +179,7 @@ func readHandshake(rd io.Reader) (host string, err error) {
func GetHttpsHostname(c frpNet.Conn) (sc frpNet.Conn, _ map[string]string, err error) {
reqInfoMap := make(map[string]string, 0)
sc, rd := newShareConn(c)
sc, rd := frpNet.NewShareConn(c)
host, err := readHandshake(rd)
if err != nil {
return sc, reqInfoMap, err

63
utils/vhost/resource.go Normal file
View File

@@ -0,0 +1,63 @@
// Copyright 2017 fatedier, fatedier@gmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package vhost
import (
"io/ioutil"
"net/http"
"strings"
"github.com/fatedier/frp/utils/version"
)
const (
NotFound = `<!DOCTYPE html>
<html>
<head>
<title>Not Found</title>
<style>
body {
width: 35em;
margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif;
}
</style>
</head>
<body>
<h1>The page you visit not found.</h1>
<p>Sorry, the page you are looking for is currently unavailable.<br/>
Please try again later.</p>
<p>The server is powered by <a href="https://github.com/fatedier/frp">frp</a>.</p>
<p><em>Faithfully yours, frp.</em></p>
</body>
</html>
`
)
func notFoundResponse() *http.Response {
header := make(http.Header)
header.Set("server", "frp/"+version.Full())
header.Set("Content-Type", "text/html")
res := &http.Response{
Status: "Not Found",
StatusCode: 404,
Proto: "HTTP/1.0",
ProtoMajor: 1,
ProtoMinor: 0,
Header: header,
Body: ioutil.NopCloser(strings.NewReader(NotFound)),
}
return res
}

View File

@@ -13,9 +13,7 @@
package vhost
import (
"bytes"
"fmt"
"io"
"strings"
"sync"
"time"
@@ -128,7 +126,7 @@ func (v *VhostMuxer) handle(c frpNet.Conn) {
sConn, reqInfoMap, err := v.vhostFunc(c)
if err != nil {
log.Error("get hostname from http/https request error: %v", err)
log.Warn("get hostname from http/https request error: %v", err)
c.Close()
return
}
@@ -137,17 +135,19 @@ func (v *VhostMuxer) handle(c frpNet.Conn) {
path := strings.ToLower(reqInfoMap["Path"])
l, ok := v.getListener(name, path)
if !ok {
res := notFoundResponse()
res.Write(c)
log.Debug("http request for host [%s] path [%s] not found", name, path)
c.Close()
return
}
// if authFunc is exist and userName/password is set
// verify user access
// then verify user access
if l.mux.authFunc != nil && l.userName != "" && l.passWord != "" {
bAccess, err := l.mux.authFunc(c, l.userName, l.passWord, reqInfoMap["Authorization"])
if bAccess == false || err != nil {
l.Debug("check Authorization failed")
l.Debug("check http Authorization failed")
res := noAuthResponse()
res.Write(c)
c.Close()
@@ -182,9 +182,10 @@ func (l *Listener) Accept() (frpNet.Conn, error) {
return nil, fmt.Errorf("Listener closed")
}
// if rewriteFunc is exist and rewriteHost is set
// if rewriteFunc is exist
// rewrite http requests with a modified host header
if l.mux.rewriteFunc != nil && l.rewriteHost != "" {
// if l.rewriteHost is empty, nothing to do
if l.mux.rewriteFunc != nil {
sConn, err := l.mux.rewriteFunc(conn, l.rewriteHost)
if err != nil {
l.Warn("host header rewrite failed: %v", err)
@@ -209,45 +210,3 @@ func (l *Listener) Close() error {
func (l *Listener) Name() string {
return l.name
}
type sharedConn struct {
frpNet.Conn
sync.Mutex
buff *bytes.Buffer
}
// the bytes you read in io.Reader, will be reserved in sharedConn
func newShareConn(conn frpNet.Conn) (*sharedConn, io.Reader) {
sc := &sharedConn{
Conn: conn,
buff: bytes.NewBuffer(make([]byte, 0, 1024)),
}
return sc, io.TeeReader(conn, sc.buff)
}
func (sc *sharedConn) Read(p []byte) (n int, err error) {
sc.Lock()
if sc.buff == nil {
sc.Unlock()
return sc.Conn.Read(p)
}
sc.Unlock()
n, err = sc.buff.Read(p)
if err == io.EOF {
sc.Lock()
sc.buff = nil
sc.Unlock()
var n2 int
n2, err = sc.Conn.Read(p[n:])
n += n2
}
return
}
func (sc *sharedConn) WriteBuff(buffer []byte) (err error) {
sc.buff.Reset()
_, err = sc.buff.Write(buffer)
return err
}

22
vendor/github.com/armon/go-socks5/.gitignore generated vendored Normal file
View File

@@ -0,0 +1,22 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe

4
vendor/github.com/armon/go-socks5/.travis.yml generated vendored Normal file
View File

@@ -0,0 +1,4 @@
language: go
go:
- 1.1
- tip

20
vendor/github.com/armon/go-socks5/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,20 @@
The MIT License (MIT)
Copyright (c) 2014 Armon Dadgar
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

45
vendor/github.com/armon/go-socks5/README.md generated vendored Normal file
View File

@@ -0,0 +1,45 @@
go-socks5 [![Build Status](https://travis-ci.org/armon/go-socks5.png)](https://travis-ci.org/armon/go-socks5)
=========
Provides the `socks5` package that implements a [SOCKS5 server](http://en.wikipedia.org/wiki/SOCKS).
SOCKS (Secure Sockets) is used to route traffic between a client and server through
an intermediate proxy layer. This can be used to bypass firewalls or NATs.
Feature
=======
The package has the following features:
* "No Auth" mode
* User/Password authentication
* Support for the CONNECT command
* Rules to do granular filtering of commands
* Custom DNS resolution
* Unit tests
TODO
====
The package still needs the following:
* Support for the BIND command
* Support for the ASSOCIATE command
Example
=======
Below is a simple example of usage
```go
// Create a SOCKS5 server
conf := &socks5.Config{}
server, err := socks5.New(conf)
if err != nil {
panic(err)
}
// Create SOCKS5 proxy on localhost port 8000
if err := server.ListenAndServe("tcp", "127.0.0.1:8000"); err != nil {
panic(err)
}
```

151
vendor/github.com/armon/go-socks5/auth.go generated vendored Normal file
View File

@@ -0,0 +1,151 @@
package socks5
import (
"fmt"
"io"
)
const (
NoAuth = uint8(0)
noAcceptable = uint8(255)
UserPassAuth = uint8(2)
userAuthVersion = uint8(1)
authSuccess = uint8(0)
authFailure = uint8(1)
)
var (
UserAuthFailed = fmt.Errorf("User authentication failed")
NoSupportedAuth = fmt.Errorf("No supported authentication mechanism")
)
// A Request encapsulates authentication state provided
// during negotiation
type AuthContext struct {
// Provided auth method
Method uint8
// Payload provided during negotiation.
// Keys depend on the used auth method.
// For UserPassauth contains Username
Payload map[string]string
}
type Authenticator interface {
Authenticate(reader io.Reader, writer io.Writer) (*AuthContext, error)
GetCode() uint8
}
// NoAuthAuthenticator is used to handle the "No Authentication" mode
type NoAuthAuthenticator struct{}
func (a NoAuthAuthenticator) GetCode() uint8 {
return NoAuth
}
func (a NoAuthAuthenticator) Authenticate(reader io.Reader, writer io.Writer) (*AuthContext, error) {
_, err := writer.Write([]byte{socks5Version, NoAuth})
return &AuthContext{NoAuth, nil}, err
}
// UserPassAuthenticator is used to handle username/password based
// authentication
type UserPassAuthenticator struct {
Credentials CredentialStore
}
func (a UserPassAuthenticator) GetCode() uint8 {
return UserPassAuth
}
func (a UserPassAuthenticator) Authenticate(reader io.Reader, writer io.Writer) (*AuthContext, error) {
// Tell the client to use user/pass auth
if _, err := writer.Write([]byte{socks5Version, UserPassAuth}); err != nil {
return nil, err
}
// Get the version and username length
header := []byte{0, 0}
if _, err := io.ReadAtLeast(reader, header, 2); err != nil {
return nil, err
}
// Ensure we are compatible
if header[0] != userAuthVersion {
return nil, fmt.Errorf("Unsupported auth version: %v", header[0])
}
// Get the user name
userLen := int(header[1])
user := make([]byte, userLen)
if _, err := io.ReadAtLeast(reader, user, userLen); err != nil {
return nil, err
}
// Get the password length
if _, err := reader.Read(header[:1]); err != nil {
return nil, err
}
// Get the password
passLen := int(header[0])
pass := make([]byte, passLen)
if _, err := io.ReadAtLeast(reader, pass, passLen); err != nil {
return nil, err
}
// Verify the password
if a.Credentials.Valid(string(user), string(pass)) {
if _, err := writer.Write([]byte{userAuthVersion, authSuccess}); err != nil {
return nil, err
}
} else {
if _, err := writer.Write([]byte{userAuthVersion, authFailure}); err != nil {
return nil, err
}
return nil, UserAuthFailed
}
// Done
return &AuthContext{UserPassAuth, map[string]string{"Username": string(user)}}, nil
}
// authenticate is used to handle connection authentication
func (s *Server) authenticate(conn io.Writer, bufConn io.Reader) (*AuthContext, error) {
// Get the methods
methods, err := readMethods(bufConn)
if err != nil {
return nil, fmt.Errorf("Failed to get auth methods: %v", err)
}
// Select a usable method
for _, method := range methods {
cator, found := s.authMethods[method]
if found {
return cator.Authenticate(bufConn, conn)
}
}
// No usable method found
return nil, noAcceptableAuth(conn)
}
// noAcceptableAuth is used to handle when we have no eligible
// authentication mechanism
func noAcceptableAuth(conn io.Writer) error {
conn.Write([]byte{socks5Version, noAcceptable})
return NoSupportedAuth
}
// readMethods is used to read the number of methods
// and proceeding auth methods
func readMethods(r io.Reader) ([]byte, error) {
header := []byte{0}
if _, err := r.Read(header); err != nil {
return nil, err
}
numMethods := int(header[0])
methods := make([]byte, numMethods)
_, err := io.ReadAtLeast(r, methods, numMethods)
return methods, err
}

17
vendor/github.com/armon/go-socks5/credentials.go generated vendored Normal file
View File

@@ -0,0 +1,17 @@
package socks5
// CredentialStore is used to support user/pass authentication
type CredentialStore interface {
Valid(user, password string) bool
}
// StaticCredentials enables using a map directly as a credential store
type StaticCredentials map[string]string
func (s StaticCredentials) Valid(user, password string) bool {
pass, ok := s[user]
if !ok {
return false
}
return password == pass
}

364
vendor/github.com/armon/go-socks5/request.go generated vendored Normal file
View File

@@ -0,0 +1,364 @@
package socks5
import (
"fmt"
"io"
"net"
"strconv"
"strings"
"golang.org/x/net/context"
)
const (
ConnectCommand = uint8(1)
BindCommand = uint8(2)
AssociateCommand = uint8(3)
ipv4Address = uint8(1)
fqdnAddress = uint8(3)
ipv6Address = uint8(4)
)
const (
successReply uint8 = iota
serverFailure
ruleFailure
networkUnreachable
hostUnreachable
connectionRefused
ttlExpired
commandNotSupported
addrTypeNotSupported
)
var (
unrecognizedAddrType = fmt.Errorf("Unrecognized address type")
)
// AddressRewriter is used to rewrite a destination transparently
type AddressRewriter interface {
Rewrite(ctx context.Context, request *Request) (context.Context, *AddrSpec)
}
// AddrSpec is used to return the target AddrSpec
// which may be specified as IPv4, IPv6, or a FQDN
type AddrSpec struct {
FQDN string
IP net.IP
Port int
}
func (a *AddrSpec) String() string {
if a.FQDN != "" {
return fmt.Sprintf("%s (%s):%d", a.FQDN, a.IP, a.Port)
}
return fmt.Sprintf("%s:%d", a.IP, a.Port)
}
// Address returns a string suitable to dial; prefer returning IP-based
// address, fallback to FQDN
func (a AddrSpec) Address() string {
if 0 != len(a.IP) {
return net.JoinHostPort(a.IP.String(), strconv.Itoa(a.Port))
}
return net.JoinHostPort(a.FQDN, strconv.Itoa(a.Port))
}
// A Request represents request received by a server
type Request struct {
// Protocol version
Version uint8
// Requested command
Command uint8
// AuthContext provided during negotiation
AuthContext *AuthContext
// AddrSpec of the the network that sent the request
RemoteAddr *AddrSpec
// AddrSpec of the desired destination
DestAddr *AddrSpec
// AddrSpec of the actual destination (might be affected by rewrite)
realDestAddr *AddrSpec
bufConn io.Reader
}
type conn interface {
Write([]byte) (int, error)
RemoteAddr() net.Addr
}
// NewRequest creates a new Request from the tcp connection
func NewRequest(bufConn io.Reader) (*Request, error) {
// Read the version byte
header := []byte{0, 0, 0}
if _, err := io.ReadAtLeast(bufConn, header, 3); err != nil {
return nil, fmt.Errorf("Failed to get command version: %v", err)
}
// Ensure we are compatible
if header[0] != socks5Version {
return nil, fmt.Errorf("Unsupported command version: %v", header[0])
}
// Read in the destination address
dest, err := readAddrSpec(bufConn)
if err != nil {
return nil, err
}
request := &Request{
Version: socks5Version,
Command: header[1],
DestAddr: dest,
bufConn: bufConn,
}
return request, nil
}
// handleRequest is used for request processing after authentication
func (s *Server) handleRequest(req *Request, conn conn) error {
ctx := context.Background()
// Resolve the address if we have a FQDN
dest := req.DestAddr
if dest.FQDN != "" {
ctx_, addr, err := s.config.Resolver.Resolve(ctx, dest.FQDN)
if err != nil {
if err := sendReply(conn, hostUnreachable, nil); err != nil {
return fmt.Errorf("Failed to send reply: %v", err)
}
return fmt.Errorf("Failed to resolve destination '%v': %v", dest.FQDN, err)
}
ctx = ctx_
dest.IP = addr
}
// Apply any address rewrites
req.realDestAddr = req.DestAddr
if s.config.Rewriter != nil {
ctx, req.realDestAddr = s.config.Rewriter.Rewrite(ctx, req)
}
// Switch on the command
switch req.Command {
case ConnectCommand:
return s.handleConnect(ctx, conn, req)
case BindCommand:
return s.handleBind(ctx, conn, req)
case AssociateCommand:
return s.handleAssociate(ctx, conn, req)
default:
if err := sendReply(conn, commandNotSupported, nil); err != nil {
return fmt.Errorf("Failed to send reply: %v", err)
}
return fmt.Errorf("Unsupported command: %v", req.Command)
}
}
// handleConnect is used to handle a connect command
func (s *Server) handleConnect(ctx context.Context, conn conn, req *Request) error {
// Check if this is allowed
if ctx_, ok := s.config.Rules.Allow(ctx, req); !ok {
if err := sendReply(conn, ruleFailure, nil); err != nil {
return fmt.Errorf("Failed to send reply: %v", err)
}
return fmt.Errorf("Connect to %v blocked by rules", req.DestAddr)
} else {
ctx = ctx_
}
// Attempt to connect
dial := s.config.Dial
if dial == nil {
dial = func(ctx context.Context, net_, addr string) (net.Conn, error) {
return net.Dial(net_, addr)
}
}
target, err := dial(ctx, "tcp", req.realDestAddr.Address())
if err != nil {
msg := err.Error()
resp := hostUnreachable
if strings.Contains(msg, "refused") {
resp = connectionRefused
} else if strings.Contains(msg, "network is unreachable") {
resp = networkUnreachable
}
if err := sendReply(conn, resp, nil); err != nil {
return fmt.Errorf("Failed to send reply: %v", err)
}
return fmt.Errorf("Connect to %v failed: %v", req.DestAddr, err)
}
defer target.Close()
// Send success
local := target.LocalAddr().(*net.TCPAddr)
bind := AddrSpec{IP: local.IP, Port: local.Port}
if err := sendReply(conn, successReply, &bind); err != nil {
return fmt.Errorf("Failed to send reply: %v", err)
}
// Start proxying
errCh := make(chan error, 2)
go proxy(target, req.bufConn, errCh)
go proxy(conn, target, errCh)
// Wait
for i := 0; i < 2; i++ {
e := <-errCh
if e != nil {
// return from this function closes target (and conn).
return e
}
}
return nil
}
// handleBind is used to handle a connect command
func (s *Server) handleBind(ctx context.Context, conn conn, req *Request) error {
// Check if this is allowed
if ctx_, ok := s.config.Rules.Allow(ctx, req); !ok {
if err := sendReply(conn, ruleFailure, nil); err != nil {
return fmt.Errorf("Failed to send reply: %v", err)
}
return fmt.Errorf("Bind to %v blocked by rules", req.DestAddr)
} else {
ctx = ctx_
}
// TODO: Support bind
if err := sendReply(conn, commandNotSupported, nil); err != nil {
return fmt.Errorf("Failed to send reply: %v", err)
}
return nil
}
// handleAssociate is used to handle a connect command
func (s *Server) handleAssociate(ctx context.Context, conn conn, req *Request) error {
// Check if this is allowed
if ctx_, ok := s.config.Rules.Allow(ctx, req); !ok {
if err := sendReply(conn, ruleFailure, nil); err != nil {
return fmt.Errorf("Failed to send reply: %v", err)
}
return fmt.Errorf("Associate to %v blocked by rules", req.DestAddr)
} else {
ctx = ctx_
}
// TODO: Support associate
if err := sendReply(conn, commandNotSupported, nil); err != nil {
return fmt.Errorf("Failed to send reply: %v", err)
}
return nil
}
// readAddrSpec is used to read AddrSpec.
// Expects an address type byte, follwed by the address and port
func readAddrSpec(r io.Reader) (*AddrSpec, error) {
d := &AddrSpec{}
// Get the address type
addrType := []byte{0}
if _, err := r.Read(addrType); err != nil {
return nil, err
}
// Handle on a per type basis
switch addrType[0] {
case ipv4Address:
addr := make([]byte, 4)
if _, err := io.ReadAtLeast(r, addr, len(addr)); err != nil {
return nil, err
}
d.IP = net.IP(addr)
case ipv6Address:
addr := make([]byte, 16)
if _, err := io.ReadAtLeast(r, addr, len(addr)); err != nil {
return nil, err
}
d.IP = net.IP(addr)
case fqdnAddress:
if _, err := r.Read(addrType); err != nil {
return nil, err
}
addrLen := int(addrType[0])
fqdn := make([]byte, addrLen)
if _, err := io.ReadAtLeast(r, fqdn, addrLen); err != nil {
return nil, err
}
d.FQDN = string(fqdn)
default:
return nil, unrecognizedAddrType
}
// Read the port
port := []byte{0, 0}
if _, err := io.ReadAtLeast(r, port, 2); err != nil {
return nil, err
}
d.Port = (int(port[0]) << 8) | int(port[1])
return d, nil
}
// sendReply is used to send a reply message
func sendReply(w io.Writer, resp uint8, addr *AddrSpec) error {
// Format the address
var addrType uint8
var addrBody []byte
var addrPort uint16
switch {
case addr == nil:
addrType = ipv4Address
addrBody = []byte{0, 0, 0, 0}
addrPort = 0
case addr.FQDN != "":
addrType = fqdnAddress
addrBody = append([]byte{byte(len(addr.FQDN))}, addr.FQDN...)
addrPort = uint16(addr.Port)
case addr.IP.To4() != nil:
addrType = ipv4Address
addrBody = []byte(addr.IP.To4())
addrPort = uint16(addr.Port)
case addr.IP.To16() != nil:
addrType = ipv6Address
addrBody = []byte(addr.IP.To16())
addrPort = uint16(addr.Port)
default:
return fmt.Errorf("Failed to format address: %v", addr)
}
// Format the message
msg := make([]byte, 6+len(addrBody))
msg[0] = socks5Version
msg[1] = resp
msg[2] = 0 // Reserved
msg[3] = addrType
copy(msg[4:], addrBody)
msg[4+len(addrBody)] = byte(addrPort >> 8)
msg[4+len(addrBody)+1] = byte(addrPort & 0xff)
// Send the message
_, err := w.Write(msg)
return err
}
type closeWriter interface {
CloseWrite() error
}
// proxy is used to suffle data from src to destination, and sends errors
// down a dedicated channel
func proxy(dst io.Writer, src io.Reader, errCh chan error) {
_, err := io.Copy(dst, src)
if tcpConn, ok := dst.(closeWriter); ok {
tcpConn.CloseWrite()
}
errCh <- err
}

23
vendor/github.com/armon/go-socks5/resolver.go generated vendored Normal file
View File

@@ -0,0 +1,23 @@
package socks5
import (
"net"
"golang.org/x/net/context"
)
// NameResolver is used to implement custom name resolution
type NameResolver interface {
Resolve(ctx context.Context, name string) (context.Context, net.IP, error)
}
// DNSResolver uses the system DNS to resolve host names
type DNSResolver struct{}
func (d DNSResolver) Resolve(ctx context.Context, name string) (context.Context, net.IP, error) {
addr, err := net.ResolveIPAddr("ip", name)
if err != nil {
return ctx, nil, err
}
return ctx, addr.IP, err
}

41
vendor/github.com/armon/go-socks5/ruleset.go generated vendored Normal file
View File

@@ -0,0 +1,41 @@
package socks5
import (
"golang.org/x/net/context"
)
// RuleSet is used to provide custom rules to allow or prohibit actions
type RuleSet interface {
Allow(ctx context.Context, req *Request) (context.Context, bool)
}
// PermitAll returns a RuleSet which allows all types of connections
func PermitAll() RuleSet {
return &PermitCommand{true, true, true}
}
// PermitNone returns a RuleSet which disallows all types of connections
func PermitNone() RuleSet {
return &PermitCommand{false, false, false}
}
// PermitCommand is an implementation of the RuleSet which
// enables filtering supported commands
type PermitCommand struct {
EnableConnect bool
EnableBind bool
EnableAssociate bool
}
func (p *PermitCommand) Allow(ctx context.Context, req *Request) (context.Context, bool) {
switch req.Command {
case ConnectCommand:
return ctx, p.EnableConnect
case BindCommand:
return ctx, p.EnableBind
case AssociateCommand:
return ctx, p.EnableAssociate
}
return ctx, false
}

169
vendor/github.com/armon/go-socks5/socks5.go generated vendored Normal file
View File

@@ -0,0 +1,169 @@
package socks5
import (
"bufio"
"fmt"
"log"
"net"
"os"
"golang.org/x/net/context"
)
const (
socks5Version = uint8(5)
)
// Config is used to setup and configure a Server
type Config struct {
// AuthMethods can be provided to implement custom authentication
// By default, "auth-less" mode is enabled.
// For password-based auth use UserPassAuthenticator.
AuthMethods []Authenticator
// If provided, username/password authentication is enabled,
// by appending a UserPassAuthenticator to AuthMethods. If not provided,
// and AUthMethods is nil, then "auth-less" mode is enabled.
Credentials CredentialStore
// Resolver can be provided to do custom name resolution.
// Defaults to DNSResolver if not provided.
Resolver NameResolver
// Rules is provided to enable custom logic around permitting
// various commands. If not provided, PermitAll is used.
Rules RuleSet
// Rewriter can be used to transparently rewrite addresses.
// This is invoked before the RuleSet is invoked.
// Defaults to NoRewrite.
Rewriter AddressRewriter
// BindIP is used for bind or udp associate
BindIP net.IP
// Logger can be used to provide a custom log target.
// Defaults to stdout.
Logger *log.Logger
// Optional function for dialing out
Dial func(ctx context.Context, network, addr string) (net.Conn, error)
}
// Server is reponsible for accepting connections and handling
// the details of the SOCKS5 protocol
type Server struct {
config *Config
authMethods map[uint8]Authenticator
}
// New creates a new Server and potentially returns an error
func New(conf *Config) (*Server, error) {
// Ensure we have at least one authentication method enabled
if len(conf.AuthMethods) == 0 {
if conf.Credentials != nil {
conf.AuthMethods = []Authenticator{&UserPassAuthenticator{conf.Credentials}}
} else {
conf.AuthMethods = []Authenticator{&NoAuthAuthenticator{}}
}
}
// Ensure we have a DNS resolver
if conf.Resolver == nil {
conf.Resolver = DNSResolver{}
}
// Ensure we have a rule set
if conf.Rules == nil {
conf.Rules = PermitAll()
}
// Ensure we have a log target
if conf.Logger == nil {
conf.Logger = log.New(os.Stdout, "", log.LstdFlags)
}
server := &Server{
config: conf,
}
server.authMethods = make(map[uint8]Authenticator)
for _, a := range conf.AuthMethods {
server.authMethods[a.GetCode()] = a
}
return server, nil
}
// ListenAndServe is used to create a listener and serve on it
func (s *Server) ListenAndServe(network, addr string) error {
l, err := net.Listen(network, addr)
if err != nil {
return err
}
return s.Serve(l)
}
// Serve is used to serve connections from a listener
func (s *Server) Serve(l net.Listener) error {
for {
conn, err := l.Accept()
if err != nil {
return err
}
go s.ServeConn(conn)
}
return nil
}
// ServeConn is used to serve a single connection.
func (s *Server) ServeConn(conn net.Conn) error {
defer conn.Close()
bufConn := bufio.NewReader(conn)
// Read the version byte
version := []byte{0}
if _, err := bufConn.Read(version); err != nil {
s.config.Logger.Printf("[ERR] socks: Failed to get version byte: %v", err)
return err
}
// Ensure we are compatible
if version[0] != socks5Version {
err := fmt.Errorf("Unsupported SOCKS version: %v", version)
s.config.Logger.Printf("[ERR] socks: %v", err)
return err
}
// Authenticate the connection
authContext, err := s.authenticate(conn, bufConn)
if err != nil {
err = fmt.Errorf("Failed to authenticate: %v", err)
s.config.Logger.Printf("[ERR] socks: %v", err)
return err
}
request, err := NewRequest(bufConn)
if err != nil {
if err == unrecognizedAddrType {
if err := sendReply(conn, addrTypeNotSupported, nil); err != nil {
return fmt.Errorf("Failed to send reply: %v", err)
}
}
return fmt.Errorf("Failed to read destination address: %v", err)
}
request.AuthContext = authContext
if client, ok := conn.RemoteAddr().(*net.TCPAddr); ok {
request.RemoteAddr = &AddrSpec{IP: client.IP, Port: client.Port}
}
// Process the client request
if err := s.handleRequest(request, conn); err != nil {
err = fmt.Errorf("Failed to handle request: %v", err)
s.config.Logger.Printf("[ERR] socks: %v", err)
return err
}
return nil
}

24
vendor/github.com/klauspost/cpuid/.gitignore generated vendored Normal file
View File

@@ -0,0 +1,24 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof

8
vendor/github.com/klauspost/cpuid/.travis.yml generated vendored Normal file
View File

@@ -0,0 +1,8 @@
language: go
go:
- 1.3
- 1.4
- 1.5
- 1.6
- tip

22
vendor/github.com/klauspost/cpuid/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,22 @@
The MIT License (MIT)
Copyright (c) 2015 Klaus Post
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

145
vendor/github.com/klauspost/cpuid/README.md generated vendored Normal file
View File

@@ -0,0 +1,145 @@
# cpuid
Package cpuid provides information about the CPU running the current program.
CPU features are detected on startup, and kept for fast access through the life of the application.
Currently x86 / x64 (AMD64) is supported, and no external C (cgo) code is used, which should make the library very easy to use.
You can access the CPU information by accessing the shared CPU variable of the cpuid library.
Package home: https://github.com/klauspost/cpuid
[![GoDoc][1]][2] [![Build Status][3]][4]
[1]: https://godoc.org/github.com/klauspost/cpuid?status.svg
[2]: https://godoc.org/github.com/klauspost/cpuid
[3]: https://travis-ci.org/klauspost/cpuid.svg
[4]: https://travis-ci.org/klauspost/cpuid
# features
## CPU Instructions
* **CMOV** (i686 CMOV)
* **NX** (NX (No-Execute) bit)
* **AMD3DNOW** (AMD 3DNOW)
* **AMD3DNOWEXT** (AMD 3DNowExt)
* **MMX** (standard MMX)
* **MMXEXT** (SSE integer functions or AMD MMX ext)
* **SSE** (SSE functions)
* **SSE2** (P4 SSE functions)
* **SSE3** (Prescott SSE3 functions)
* **SSSE3** (Conroe SSSE3 functions)
* **SSE4** (Penryn SSE4.1 functions)
* **SSE4A** (AMD Barcelona microarchitecture SSE4a instructions)
* **SSE42** (Nehalem SSE4.2 functions)
* **AVX** (AVX functions)
* **AVX2** (AVX2 functions)
* **FMA3** (Intel FMA 3)
* **FMA4** (Bulldozer FMA4 functions)
* **XOP** (Bulldozer XOP functions)
* **F16C** (Half-precision floating-point conversion)
* **BMI1** (Bit Manipulation Instruction Set 1)
* **BMI2** (Bit Manipulation Instruction Set 2)
* **TBM** (AMD Trailing Bit Manipulation)
* **LZCNT** (LZCNT instruction)
* **POPCNT** (POPCNT instruction)
* **AESNI** (Advanced Encryption Standard New Instructions)
* **CLMUL** (Carry-less Multiplication)
* **HTT** (Hyperthreading (enabled))
* **HLE** (Hardware Lock Elision)
* **RTM** (Restricted Transactional Memory)
* **RDRAND** (RDRAND instruction is available)
* **RDSEED** (RDSEED instruction is available)
* **ADX** (Intel ADX (Multi-Precision Add-Carry Instruction Extensions))
* **SHA** (Intel SHA Extensions)
* **AVX512F** (AVX-512 Foundation)
* **AVX512DQ** (AVX-512 Doubleword and Quadword Instructions)
* **AVX512IFMA** (AVX-512 Integer Fused Multiply-Add Instructions)
* **AVX512PF** (AVX-512 Prefetch Instructions)
* **AVX512ER** (AVX-512 Exponential and Reciprocal Instructions)
* **AVX512CD** (AVX-512 Conflict Detection Instructions)
* **AVX512BW** (AVX-512 Byte and Word Instructions)
* **AVX512VL** (AVX-512 Vector Length Extensions)
* **AVX512VBMI** (AVX-512 Vector Bit Manipulation Instructions)
* **MPX** (Intel MPX (Memory Protection Extensions))
* **ERMS** (Enhanced REP MOVSB/STOSB)
* **RDTSCP** (RDTSCP Instruction)
* **CX16** (CMPXCHG16B Instruction)
* **SGX** (Software Guard Extensions, with activation details)
## Performance
* **RDTSCP()** Returns current cycle count. Can be used for benchmarking.
* **SSE2SLOW** (SSE2 is supported, but usually not faster)
* **SSE3SLOW** (SSE3 is supported, but usually not faster)
* **ATOM** (Atom processor, some SSSE3 instructions are slower)
* **Cache line** (Probable size of a cache line).
* **L1, L2, L3 Cache size** on newer Intel/AMD CPUs.
## Cpu Vendor/VM
* **Intel**
* **AMD**
* **VIA**
* **Transmeta**
* **NSC**
* **KVM** (Kernel-based Virtual Machine)
* **MSVM** (Microsoft Hyper-V or Windows Virtual PC)
* **VMware**
* **XenHVM**
# installing
```go get github.com/klauspost/cpuid```
# example
```Go
package main
import (
"fmt"
"github.com/klauspost/cpuid"
)
func main() {
// Print basic CPU information:
fmt.Println("Name:", cpuid.CPU.BrandName)
fmt.Println("PhysicalCores:", cpuid.CPU.PhysicalCores)
fmt.Println("ThreadsPerCore:", cpuid.CPU.ThreadsPerCore)
fmt.Println("LogicalCores:", cpuid.CPU.LogicalCores)
fmt.Println("Family", cpuid.CPU.Family, "Model:", cpuid.CPU.Model)
fmt.Println("Features:", cpuid.CPU.Features)
fmt.Println("Cacheline bytes:", cpuid.CPU.CacheLine)
fmt.Println("L1 Data Cache:", cpuid.CPU.Cache.L1D, "bytes")
fmt.Println("L1 Instruction Cache:", cpuid.CPU.Cache.L1D, "bytes")
fmt.Println("L2 Cache:", cpuid.CPU.Cache.L2, "bytes")
fmt.Println("L3 Cache:", cpuid.CPU.Cache.L3, "bytes")
// Test if we have a specific feature:
if cpuid.CPU.SSE() {
fmt.Println("We have Streaming SIMD Extensions")
}
}
```
Sample output:
```
>go run main.go
Name: Intel(R) Core(TM) i5-2540M CPU @ 2.60GHz
PhysicalCores: 2
ThreadsPerCore: 2
LogicalCores: 4
Family 6 Model: 42
Features: CMOV,MMX,MMXEXT,SSE,SSE2,SSE3,SSSE3,SSE4.1,SSE4.2,AVX,AESNI,CLMUL
Cacheline bytes: 64
We have Streaming SIMD Extensions
```
# private package
In the "private" folder you can find an autogenerated version of the library you can include in your own packages.
For this purpose all exports are removed, and functions and constants are lowercased.
This is not a recommended way of using the library, but provided for convenience, if it is difficult for you to use external packages.
# license
This code is published under an MIT license. See LICENSE file for more information.

1022
vendor/github.com/klauspost/cpuid/cpuid.go generated vendored Normal file

File diff suppressed because it is too large Load Diff

42
vendor/github.com/klauspost/cpuid/cpuid_386.s generated vendored Normal file
View File

@@ -0,0 +1,42 @@
// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file.
// +build 386,!gccgo
// func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32)
TEXT ·asmCpuid(SB), 7, $0
XORL CX, CX
MOVL op+0(FP), AX
CPUID
MOVL AX, eax+4(FP)
MOVL BX, ebx+8(FP)
MOVL CX, ecx+12(FP)
MOVL DX, edx+16(FP)
RET
// func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32)
TEXT ·asmCpuidex(SB), 7, $0
MOVL op+0(FP), AX
MOVL op2+4(FP), CX
CPUID
MOVL AX, eax+8(FP)
MOVL BX, ebx+12(FP)
MOVL CX, ecx+16(FP)
MOVL DX, edx+20(FP)
RET
// func xgetbv(index uint32) (eax, edx uint32)
TEXT ·asmXgetbv(SB), 7, $0
MOVL index+0(FP), CX
BYTE $0x0f; BYTE $0x01; BYTE $0xd0 // XGETBV
MOVL AX, eax+4(FP)
MOVL DX, edx+8(FP)
RET
// func asmRdtscpAsm() (eax, ebx, ecx, edx uint32)
TEXT ·asmRdtscpAsm(SB), 7, $0
BYTE $0x0F; BYTE $0x01; BYTE $0xF9 // RDTSCP
MOVL AX, eax+0(FP)
MOVL BX, ebx+4(FP)
MOVL CX, ecx+8(FP)
MOVL DX, edx+12(FP)
RET

42
vendor/github.com/klauspost/cpuid/cpuid_amd64.s generated vendored Normal file
View File

@@ -0,0 +1,42 @@
// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file.
//+build amd64,!gccgo
// func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32)
TEXT ·asmCpuid(SB), 7, $0
XORQ CX, CX
MOVL op+0(FP), AX
CPUID
MOVL AX, eax+8(FP)
MOVL BX, ebx+12(FP)
MOVL CX, ecx+16(FP)
MOVL DX, edx+20(FP)
RET
// func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32)
TEXT ·asmCpuidex(SB), 7, $0
MOVL op+0(FP), AX
MOVL op2+4(FP), CX
CPUID
MOVL AX, eax+8(FP)
MOVL BX, ebx+12(FP)
MOVL CX, ecx+16(FP)
MOVL DX, edx+20(FP)
RET
// func asmXgetbv(index uint32) (eax, edx uint32)
TEXT ·asmXgetbv(SB), 7, $0
MOVL index+0(FP), CX
BYTE $0x0f; BYTE $0x01; BYTE $0xd0 // XGETBV
MOVL AX, eax+8(FP)
MOVL DX, edx+12(FP)
RET
// func asmRdtscpAsm() (eax, ebx, ecx, edx uint32)
TEXT ·asmRdtscpAsm(SB), 7, $0
BYTE $0x0F; BYTE $0x01; BYTE $0xF9 // RDTSCP
MOVL AX, eax+0(FP)
MOVL BX, ebx+4(FP)
MOVL CX, ecx+8(FP)
MOVL DX, edx+12(FP)
RET

17
vendor/github.com/klauspost/cpuid/detect_intel.go generated vendored Normal file
View File

@@ -0,0 +1,17 @@
// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file.
// +build 386,!gccgo amd64,!gccgo
package cpuid
func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32)
func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32)
func asmXgetbv(index uint32) (eax, edx uint32)
func asmRdtscpAsm() (eax, ebx, ecx, edx uint32)
func initCPU() {
cpuid = asmCpuid
cpuidex = asmCpuidex
xgetbv = asmXgetbv
rdtscpAsm = asmRdtscpAsm
}

23
vendor/github.com/klauspost/cpuid/detect_ref.go generated vendored Normal file
View File

@@ -0,0 +1,23 @@
// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file.
// +build !amd64,!386 gccgo
package cpuid
func initCPU() {
cpuid = func(op uint32) (eax, ebx, ecx, edx uint32) {
return 0, 0, 0, 0
}
cpuidex = func(op, op2 uint32) (eax, ebx, ecx, edx uint32) {
return 0, 0, 0, 0
}
xgetbv = func(index uint32) (eax, edx uint32) {
return 0, 0
}
rdtscpAsm = func() (eax, ebx, ecx, edx uint32) {
return 0, 0, 0, 0
}
}

3
vendor/github.com/klauspost/cpuid/generate.go generated vendored Normal file
View File

@@ -0,0 +1,3 @@
package cpuid
//go:generate go run private-gen.go

476
vendor/github.com/klauspost/cpuid/private-gen.go generated vendored Normal file
View File

@@ -0,0 +1,476 @@
// +build ignore
package main
import (
"bytes"
"fmt"
"go/ast"
"go/parser"
"go/printer"
"go/token"
"io"
"io/ioutil"
"log"
"os"
"reflect"
"strings"
"unicode"
"unicode/utf8"
)
var inFiles = []string{"cpuid.go", "cpuid_test.go"}
var copyFiles = []string{"cpuid_amd64.s", "cpuid_386.s", "detect_ref.go", "detect_intel.go"}
var fileSet = token.NewFileSet()
var reWrites = []rewrite{
initRewrite("CPUInfo -> cpuInfo"),
initRewrite("Vendor -> vendor"),
initRewrite("Flags -> flags"),
initRewrite("Detect -> detect"),
initRewrite("CPU -> cpu"),
}
var excludeNames = map[string]bool{"string": true, "join": true, "trim": true,
// cpuid_test.go
"t": true, "println": true, "logf": true, "log": true, "fatalf": true, "fatal": true,
}
var excludePrefixes = []string{"test", "benchmark"}
func main() {
Package := "private"
parserMode := parser.ParseComments
exported := make(map[string]rewrite)
for _, file := range inFiles {
in, err := os.Open(file)
if err != nil {
log.Fatalf("opening input", err)
}
src, err := ioutil.ReadAll(in)
if err != nil {
log.Fatalf("reading input", err)
}
astfile, err := parser.ParseFile(fileSet, file, src, parserMode)
if err != nil {
log.Fatalf("parsing input", err)
}
for _, rw := range reWrites {
astfile = rw(astfile)
}
// Inspect the AST and print all identifiers and literals.
var startDecl token.Pos
var endDecl token.Pos
ast.Inspect(astfile, func(n ast.Node) bool {
var s string
switch x := n.(type) {
case *ast.Ident:
if x.IsExported() {
t := strings.ToLower(x.Name)
for _, pre := range excludePrefixes {
if strings.HasPrefix(t, pre) {
return true
}
}
if excludeNames[t] != true {
//if x.Pos() > startDecl && x.Pos() < endDecl {
exported[x.Name] = initRewrite(x.Name + " -> " + t)
}
}
case *ast.GenDecl:
if x.Tok == token.CONST && x.Lparen > 0 {
startDecl = x.Lparen
endDecl = x.Rparen
// fmt.Printf("Decl:%s -> %s\n", fileSet.Position(startDecl), fileSet.Position(endDecl))
}
}
if s != "" {
fmt.Printf("%s:\t%s\n", fileSet.Position(n.Pos()), s)
}
return true
})
for _, rw := range exported {
astfile = rw(astfile)
}
var buf bytes.Buffer
printer.Fprint(&buf, fileSet, astfile)
// Remove package documentation and insert information
s := buf.String()
ind := strings.Index(buf.String(), "\npackage cpuid")
s = s[ind:]
s = "// Generated, DO NOT EDIT,\n" +
"// but copy it to your own project and rename the package.\n" +
"// See more at http://github.com/klauspost/cpuid\n" +
s
outputName := Package + string(os.PathSeparator) + file
err = ioutil.WriteFile(outputName, []byte(s), 0644)
if err != nil {
log.Fatalf("writing output: %s", err)
}
log.Println("Generated", outputName)
}
for _, file := range copyFiles {
dst := ""
if strings.HasPrefix(file, "cpuid") {
dst = Package + string(os.PathSeparator) + file
} else {
dst = Package + string(os.PathSeparator) + "cpuid_" + file
}
err := copyFile(file, dst)
if err != nil {
log.Fatalf("copying file: %s", err)
}
log.Println("Copied", dst)
}
}
// CopyFile copies a file from src to dst. If src and dst files exist, and are
// the same, then return success. Copy the file contents from src to dst.
func copyFile(src, dst string) (err error) {
sfi, err := os.Stat(src)
if err != nil {
return
}
if !sfi.Mode().IsRegular() {
// cannot copy non-regular files (e.g., directories,
// symlinks, devices, etc.)
return fmt.Errorf("CopyFile: non-regular source file %s (%q)", sfi.Name(), sfi.Mode().String())
}
dfi, err := os.Stat(dst)
if err != nil {
if !os.IsNotExist(err) {
return
}
} else {
if !(dfi.Mode().IsRegular()) {
return fmt.Errorf("CopyFile: non-regular destination file %s (%q)", dfi.Name(), dfi.Mode().String())
}
if os.SameFile(sfi, dfi) {
return
}
}
err = copyFileContents(src, dst)
return
}
// copyFileContents copies the contents of the file named src to the file named
// by dst. The file will be created if it does not already exist. If the
// destination file exists, all it's contents will be replaced by the contents
// of the source file.
func copyFileContents(src, dst string) (err error) {
in, err := os.Open(src)
if err != nil {
return
}
defer in.Close()
out, err := os.Create(dst)
if err != nil {
return
}
defer func() {
cerr := out.Close()
if err == nil {
err = cerr
}
}()
if _, err = io.Copy(out, in); err != nil {
return
}
err = out.Sync()
return
}
type rewrite func(*ast.File) *ast.File
// Mostly copied from gofmt
func initRewrite(rewriteRule string) rewrite {
f := strings.Split(rewriteRule, "->")
if len(f) != 2 {
fmt.Fprintf(os.Stderr, "rewrite rule must be of the form 'pattern -> replacement'\n")
os.Exit(2)
}
pattern := parseExpr(f[0], "pattern")
replace := parseExpr(f[1], "replacement")
return func(p *ast.File) *ast.File { return rewriteFile(pattern, replace, p) }
}
// parseExpr parses s as an expression.
// It might make sense to expand this to allow statement patterns,
// but there are problems with preserving formatting and also
// with what a wildcard for a statement looks like.
func parseExpr(s, what string) ast.Expr {
x, err := parser.ParseExpr(s)
if err != nil {
fmt.Fprintf(os.Stderr, "parsing %s %s at %s\n", what, s, err)
os.Exit(2)
}
return x
}
// Keep this function for debugging.
/*
func dump(msg string, val reflect.Value) {
fmt.Printf("%s:\n", msg)
ast.Print(fileSet, val.Interface())
fmt.Println()
}
*/
// rewriteFile applies the rewrite rule 'pattern -> replace' to an entire file.
func rewriteFile(pattern, replace ast.Expr, p *ast.File) *ast.File {
cmap := ast.NewCommentMap(fileSet, p, p.Comments)
m := make(map[string]reflect.Value)
pat := reflect.ValueOf(pattern)
repl := reflect.ValueOf(replace)
var rewriteVal func(val reflect.Value) reflect.Value
rewriteVal = func(val reflect.Value) reflect.Value {
// don't bother if val is invalid to start with
if !val.IsValid() {
return reflect.Value{}
}
for k := range m {
delete(m, k)
}
val = apply(rewriteVal, val)
if match(m, pat, val) {
val = subst(m, repl, reflect.ValueOf(val.Interface().(ast.Node).Pos()))
}
return val
}
r := apply(rewriteVal, reflect.ValueOf(p)).Interface().(*ast.File)
r.Comments = cmap.Filter(r).Comments() // recreate comments list
return r
}
// set is a wrapper for x.Set(y); it protects the caller from panics if x cannot be changed to y.
func set(x, y reflect.Value) {
// don't bother if x cannot be set or y is invalid
if !x.CanSet() || !y.IsValid() {
return
}
defer func() {
if x := recover(); x != nil {
if s, ok := x.(string); ok &&
(strings.Contains(s, "type mismatch") || strings.Contains(s, "not assignable")) {
// x cannot be set to y - ignore this rewrite
return
}
panic(x)
}
}()
x.Set(y)
}
// Values/types for special cases.
var (
objectPtrNil = reflect.ValueOf((*ast.Object)(nil))
scopePtrNil = reflect.ValueOf((*ast.Scope)(nil))
identType = reflect.TypeOf((*ast.Ident)(nil))
objectPtrType = reflect.TypeOf((*ast.Object)(nil))
positionType = reflect.TypeOf(token.NoPos)
callExprType = reflect.TypeOf((*ast.CallExpr)(nil))
scopePtrType = reflect.TypeOf((*ast.Scope)(nil))
)
// apply replaces each AST field x in val with f(x), returning val.
// To avoid extra conversions, f operates on the reflect.Value form.
func apply(f func(reflect.Value) reflect.Value, val reflect.Value) reflect.Value {
if !val.IsValid() {
return reflect.Value{}
}
// *ast.Objects introduce cycles and are likely incorrect after
// rewrite; don't follow them but replace with nil instead
if val.Type() == objectPtrType {
return objectPtrNil
}
// similarly for scopes: they are likely incorrect after a rewrite;
// replace them with nil
if val.Type() == scopePtrType {
return scopePtrNil
}
switch v := reflect.Indirect(val); v.Kind() {
case reflect.Slice:
for i := 0; i < v.Len(); i++ {
e := v.Index(i)
set(e, f(e))
}
case reflect.Struct:
for i := 0; i < v.NumField(); i++ {
e := v.Field(i)
set(e, f(e))
}
case reflect.Interface:
e := v.Elem()
set(v, f(e))
}
return val
}
func isWildcard(s string) bool {
rune, size := utf8.DecodeRuneInString(s)
return size == len(s) && unicode.IsLower(rune)
}
// match returns true if pattern matches val,
// recording wildcard submatches in m.
// If m == nil, match checks whether pattern == val.
func match(m map[string]reflect.Value, pattern, val reflect.Value) bool {
// Wildcard matches any expression. If it appears multiple
// times in the pattern, it must match the same expression
// each time.
if m != nil && pattern.IsValid() && pattern.Type() == identType {
name := pattern.Interface().(*ast.Ident).Name
if isWildcard(name) && val.IsValid() {
// wildcards only match valid (non-nil) expressions.
if _, ok := val.Interface().(ast.Expr); ok && !val.IsNil() {
if old, ok := m[name]; ok {
return match(nil, old, val)
}
m[name] = val
return true
}
}
}
// Otherwise, pattern and val must match recursively.
if !pattern.IsValid() || !val.IsValid() {
return !pattern.IsValid() && !val.IsValid()
}
if pattern.Type() != val.Type() {
return false
}
// Special cases.
switch pattern.Type() {
case identType:
// For identifiers, only the names need to match
// (and none of the other *ast.Object information).
// This is a common case, handle it all here instead
// of recursing down any further via reflection.
p := pattern.Interface().(*ast.Ident)
v := val.Interface().(*ast.Ident)
return p == nil && v == nil || p != nil && v != nil && p.Name == v.Name
case objectPtrType, positionType:
// object pointers and token positions always match
return true
case callExprType:
// For calls, the Ellipsis fields (token.Position) must
// match since that is how f(x) and f(x...) are different.
// Check them here but fall through for the remaining fields.
p := pattern.Interface().(*ast.CallExpr)
v := val.Interface().(*ast.CallExpr)
if p.Ellipsis.IsValid() != v.Ellipsis.IsValid() {
return false
}
}
p := reflect.Indirect(pattern)
v := reflect.Indirect(val)
if !p.IsValid() || !v.IsValid() {
return !p.IsValid() && !v.IsValid()
}
switch p.Kind() {
case reflect.Slice:
if p.Len() != v.Len() {
return false
}
for i := 0; i < p.Len(); i++ {
if !match(m, p.Index(i), v.Index(i)) {
return false
}
}
return true
case reflect.Struct:
for i := 0; i < p.NumField(); i++ {
if !match(m, p.Field(i), v.Field(i)) {
return false
}
}
return true
case reflect.Interface:
return match(m, p.Elem(), v.Elem())
}
// Handle token integers, etc.
return p.Interface() == v.Interface()
}
// subst returns a copy of pattern with values from m substituted in place
// of wildcards and pos used as the position of tokens from the pattern.
// if m == nil, subst returns a copy of pattern and doesn't change the line
// number information.
func subst(m map[string]reflect.Value, pattern reflect.Value, pos reflect.Value) reflect.Value {
if !pattern.IsValid() {
return reflect.Value{}
}
// Wildcard gets replaced with map value.
if m != nil && pattern.Type() == identType {
name := pattern.Interface().(*ast.Ident).Name
if isWildcard(name) {
if old, ok := m[name]; ok {
return subst(nil, old, reflect.Value{})
}
}
}
if pos.IsValid() && pattern.Type() == positionType {
// use new position only if old position was valid in the first place
if old := pattern.Interface().(token.Pos); !old.IsValid() {
return pattern
}
return pos
}
// Otherwise copy.
switch p := pattern; p.Kind() {
case reflect.Slice:
v := reflect.MakeSlice(p.Type(), p.Len(), p.Len())
for i := 0; i < p.Len(); i++ {
v.Index(i).Set(subst(m, p.Index(i), pos))
}
return v
case reflect.Struct:
v := reflect.New(p.Type()).Elem()
for i := 0; i < p.NumField(); i++ {
v.Field(i).Set(subst(m, p.Field(i), pos))
}
return v
case reflect.Ptr:
v := reflect.New(p.Type()).Elem()
if elem := p.Elem(); elem.IsValid() {
v.Set(subst(m, elem, pos).Addr())
}
return v
case reflect.Interface:
v := reflect.New(p.Type()).Elem()
if elem := p.Elem(); elem.IsValid() {
v.Set(subst(m, elem, pos))
}
return v
}
return pattern
}

24
vendor/github.com/klauspost/reedsolomon/.gitignore generated vendored Normal file
View File

@@ -0,0 +1,24 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof

33
vendor/github.com/klauspost/reedsolomon/.travis.yml generated vendored Normal file
View File

@@ -0,0 +1,33 @@
language: go
sudo: false
os:
- linux
- osx
go:
- 1.5
- 1.6
- 1.7
- 1.8
- master
install:
- go get ./...
script:
- go vet ./...
- go test -v -cpu=1,2,4 .
- go test -v -cpu=1,2,4 -short -race .
- go test -tags=noasm -v -cpu=1,2,4 -short -race .
- go build examples/simple-decoder.go
- go build examples/simple-encoder.go
- go build examples/stream-decoder.go
- go build examples/stream-encoder.go
- diff <(gofmt -d .) <("")
matrix:
allow_failures:
- go: 'master'
fast_finish: true

23
vendor/github.com/klauspost/reedsolomon/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,23 @@
The MIT License (MIT)
Copyright (c) 2015 Klaus Post
Copyright (c) 2015 Backblaze
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

216
vendor/github.com/klauspost/reedsolomon/README.md generated vendored Normal file
View File

@@ -0,0 +1,216 @@
# Reed-Solomon
[![GoDoc][1]][2] [![Build Status][3]][4]
[1]: https://godoc.org/github.com/klauspost/reedsolomon?status.svg
[2]: https://godoc.org/github.com/klauspost/reedsolomon
[3]: https://travis-ci.org/klauspost/reedsolomon.svg?branch=master
[4]: https://travis-ci.org/klauspost/reedsolomon
Reed-Solomon Erasure Coding in Go, with speeds exceeding 1GB/s/cpu core implemented in pure Go.
This is a golang port of the [JavaReedSolomon](https://github.com/Backblaze/JavaReedSolomon) library released by [Backblaze](http://backblaze.com), with some additional optimizations.
For an introduction on erasure coding, see the post on the [Backblaze blog](https://www.backblaze.com/blog/reed-solomon/).
Package home: https://github.com/klauspost/reedsolomon
Godoc: https://godoc.org/github.com/klauspost/reedsolomon
# Installation
To get the package use the standard:
```bash
go get github.com/klauspost/reedsolomon
```
# Usage
This section assumes you know the basics of Reed-Solomon encoding. A good start is this [Backblaze blog post](https://www.backblaze.com/blog/reed-solomon/).
This package performs the calculation of the parity sets. The usage is therefore relatively simple.
First of all, you need to choose your distribution of data and parity shards. A 'good' distribution is very subjective, and will depend a lot on your usage scenario. A good starting point is above 5 and below 257 data shards (the maximum supported number), and the number of parity shards to be 2 or above, and below the number of data shards.
To create an encoder with 10 data shards (where your data goes) and 3 parity shards (calculated):
```Go
enc, err := reedsolomon.New(10, 3)
```
This encoder will work for all parity sets with this distribution of data and parity shards. The error will only be set if you specify 0 or negative values in any of the parameters, or if you specify more than 256 data shards.
The you send and receive data is a simple slice of byte slices; `[][]byte`. In the example above, the top slice must have a length of 13.
```Go
data := make([][]byte, 13)
```
You should then fill the 10 first slices with *equally sized* data, and create parity shards that will be populated with parity data. In this case we create the data in memory, but you could for instance also use [mmap](https://github.com/edsrzf/mmap-go) to map files.
```Go
// Create all shards, size them at 50000 each
for i := range input {
data[i] := make([]byte, 50000)
}
// Fill some data into the data shards
for i, in := range data[:10] {
for j:= range in {
in[j] = byte((i+j)&0xff)
}
}
```
To populate the parity shards, you simply call `Encode()` with your data.
```Go
err = enc.Encode(data)
```
The only cases where you should get an error is, if the data shards aren't of equal size. The last 3 shards now contain parity data. You can verify this by calling `Verify()`:
```Go
ok, err = enc.Verify(data)
```
The final (and important) part is to be able to reconstruct missing shards. For this to work, you need to know which parts of your data is missing. The encoder *does not know which parts are invalid*, so if data corruption is a likely scenario, you need to implement a hash check for each shard. If a byte has changed in your set, and you don't know which it is, there is no way to reconstruct the data set.
To indicate missing data, you set the shard to nil before calling `Reconstruct()`:
```Go
// Delete two data shards
data[3] = nil
data[7] = nil
// Reconstruct the missing shards
err := enc.Reconstruct(data)
```
The missing data and parity shards will be recreated. If more than 3 shards are missing, the reconstruction will fail.
So to sum up reconstruction:
* The number of data/parity shards must match the numbers used for encoding.
* The order of shards must be the same as used when encoding.
* You may only supply data you know is valid.
* Invalid shards should be set to nil.
For complete examples of an encoder and decoder see the [examples folder](https://github.com/klauspost/reedsolomon/tree/master/examples).
# Splitting/Joining Data
You might have a large slice of data. To help you split this, there are some helper functions that can split and join a single byte slice.
```Go
bigfile, _ := ioutil.Readfile("myfile.data")
// Split the file
split, err := enc.Split(bigfile)
```
This will split the file into the number of data shards set when creating the encoder and create empty parity shards.
An important thing to note is that you have to *keep track of the exact input size*. If the size of the input isn't divisible by the number of data shards, extra zeros will be inserted in the last shard.
To join a data set, use the `Join()` function, which will join the shards and write it to the `io.Writer` you supply:
```Go
// Join a data set and write it to io.Discard.
err = enc.Join(io.Discard, data, len(bigfile))
```
# Streaming/Merging
It might seem like a limitation that all data should be in memory, but an important property is that *as long as the number of data/parity shards are the same, you can merge/split data sets*, and they will remain valid as a separate set.
```Go
// Split the data set of 50000 elements into two of 25000
splitA := make([][]byte, 13)
splitB := make([][]byte, 13)
// Merge into a 100000 element set
merged := make([][]byte, 13)
for i := range data {
splitA[i] = data[i][:25000]
splitB[i] = data[i][25000:]
// Concencate it to itself
merged[i] = append(make([]byte, 0, len(data[i])*2), data[i]...)
merged[i] = append(merged[i], data[i]...)
}
// Each part should still verify as ok.
ok, err := enc.Verify(splitA)
if ok && err == nil {
log.Println("splitA ok")
}
ok, err = enc.Verify(splitB)
if ok && err == nil {
log.Println("splitB ok")
}
ok, err = enc.Verify(merge)
if ok && err == nil {
log.Println("merge ok")
}
```
This means that if you have a data set that may not fit into memory, you can split processing into smaller blocks. For the best throughput, don't use too small blocks.
This also means that you can divide big input up into smaller blocks, and do reconstruction on parts of your data. This doesn't give the same flexibility of a higher number of data shards, but it will be much more performant.
# Streaming API
There has been added support for a streaming API, to help perform fully streaming operations, which enables you to do the same operations, but on streams. To use the stream API, use [`NewStream`](https://godoc.org/github.com/klauspost/reedsolomon#NewStream) function to create the encoding/decoding interfaces. You can use [`NewStreamC`](https://godoc.org/github.com/klauspost/reedsolomon#NewStreamC) to ready an interface that reads/writes concurrently from the streams.
Input is delivered as `[]io.Reader`, output as `[]io.Writer`, and functionality corresponds to the in-memory API. Each stream must supply the same amount of data, similar to how each slice must be similar size with the in-memory API.
If an error occurs in relation to a stream, a [`StreamReadError`](https://godoc.org/github.com/klauspost/reedsolomon#StreamReadError) or [`StreamWriteError`](https://godoc.org/github.com/klauspost/reedsolomon#StreamWriteError) will help you determine which stream was the offender.
There is no buffering or timeouts/retry specified. If you want to add that, you need to add it to the Reader/Writer.
For complete examples of a streaming encoder and decoder see the [examples folder](https://github.com/klauspost/reedsolomon/tree/master/examples).
#Advanced Options
You can modify internal options which affects how jobs are split between and processed by goroutines.
To create options, use the WithXXX functions. You can supply options to `New`, `NewStream` and `NewStreamC`. If no Options are supplied, default options are used.
Example of how to supply options:
```Go
enc, err := reedsolomon.New(10, 3, WithMaxGoroutines(25))
```
# Performance
Performance depends mainly on the number of parity shards. In rough terms, doubling the number of parity shards will double the encoding time.
Here are the throughput numbers with some different selections of data and parity shards. For reference each shard is 1MB random data, and 2 CPU cores are used for encoding.
| Data | Parity | Parity | MB/s | SSSE3 MB/s | SSSE3 Speed | Rel. Speed |
|------|--------|--------|--------|-------------|-------------|------------|
| 5 | 2 | 40% | 576,11 | 2599,2 | 451% | 100,00% |
| 10 | 2 | 20% | 587,73 | 3100,28 | 528% | 102,02% |
| 10 | 4 | 40% | 298,38 | 2470,97 | 828% | 51,79% |
| 50 | 20 | 40% | 59,81 | 713,28 | 1193% | 10,38% |
If `runtime.GOMAXPROCS()` is set to a value higher than 1, the encoder will use multiple goroutines to perform the calculations in `Verify`, `Encode` and `Reconstruct`.
Example of performance scaling on Intel(R) Core(TM) i7-2600 CPU @ 3.40GHz - 4 physical cores, 8 logical cores. The example uses 10 blocks with 16MB data each and 4 parity blocks.
| Threads | MB/s | Speed |
|---------|---------|-------|
| 1 | 1355,11 | 100% |
| 2 | 2339,78 | 172% |
| 4 | 3179,33 | 235% |
| 8 | 4346,18 | 321% |
# asm2plan9s
[asm2plan9s](https://github.com/fwessels/asm2plan9s) is used for assembling the AVX2 instructions into their BYTE/WORD/LONG equivalents.
# Links
* [Backblaze Open Sources Reed-Solomon Erasure Coding Source Code](https://www.backblaze.com/blog/reed-solomon/).
* [JavaReedSolomon](https://github.com/Backblaze/JavaReedSolomon). Compatible java library by Backblaze.
* [reedsolomon-c](https://github.com/jannson/reedsolomon-c). C version, compatible with output from this package.
* [Reed-Solomon Erasure Coding in Haskell](https://github.com/NicolasT/reedsolomon). Haskell port of the package with similar performance.
* [go-erasure](https://github.com/somethingnew2-0/go-erasure). A similar library using cgo, slower in my tests.
* [rsraid](https://github.com/goayame/rsraid). A similar library written in Go. Slower, but supports more shards.
* [Screaming Fast Galois Field Arithmetic](http://www.snia.org/sites/default/files2/SDC2013/presentations/NewThinking/EthanMiller_Screaming_Fast_Galois_Field%20Arithmetic_SIMD%20Instructions.pdf). Basis for SSE3 optimizations.
# License
This code, as the original [JavaReedSolomon](https://github.com/Backblaze/JavaReedSolomon) is published under an MIT license. See LICENSE file for more information.

20
vendor/github.com/klauspost/reedsolomon/appveyor.yml generated vendored Normal file
View File

@@ -0,0 +1,20 @@
os: Visual Studio 2015
platform: x64
clone_folder: c:\gopath\src\github.com\klauspost\reedsolomon
# environment variables
environment:
GOPATH: c:\gopath
install:
- echo %PATH%
- echo %GOPATH%
- go version
- go env
- go get -d ./...
build_script:
- go test -v -cpu=2 ./...
- go test -cpu=1,2,4 -short -race ./...

134
vendor/github.com/klauspost/reedsolomon/galois.go generated vendored Normal file

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,73 @@
//+build !noasm
//+build !appengine
// Copyright 2015, Klaus Post, see LICENSE for details.
package reedsolomon
//go:noescape
func galMulSSSE3(low, high, in, out []byte)
//go:noescape
func galMulSSSE3Xor(low, high, in, out []byte)
//go:noescape
func galMulAVX2Xor(low, high, in, out []byte)
//go:noescape
func galMulAVX2(low, high, in, out []byte)
// This is what the assembler rountes does in blocks of 16 bytes:
/*
func galMulSSSE3(low, high, in, out []byte) {
for n, input := range in {
l := input & 0xf
h := input >> 4
out[n] = low[l] ^ high[h]
}
}
func galMulSSSE3Xor(low, high, in, out []byte) {
for n, input := range in {
l := input & 0xf
h := input >> 4
out[n] ^= low[l] ^ high[h]
}
}
*/
func galMulSlice(c byte, in, out []byte, ssse3, avx2 bool) {
var done int
if avx2 {
galMulAVX2(mulTableLow[c][:], mulTableHigh[c][:], in, out)
done = (len(in) >> 5) << 5
} else if ssse3 {
galMulSSSE3(mulTableLow[c][:], mulTableHigh[c][:], in, out)
done = (len(in) >> 4) << 4
}
remain := len(in) - done
if remain > 0 {
mt := mulTable[c]
for i := done; i < len(in); i++ {
out[i] = mt[in[i]]
}
}
}
func galMulSliceXor(c byte, in, out []byte, ssse3, avx2 bool) {
var done int
if avx2 {
galMulAVX2Xor(mulTableLow[c][:], mulTableHigh[c][:], in, out)
done = (len(in) >> 5) << 5
} else if ssse3 {
galMulSSSE3Xor(mulTableLow[c][:], mulTableHigh[c][:], in, out)
done = (len(in) >> 4) << 4
}
remain := len(in) - done
if remain > 0 {
mt := mulTable[c]
for i := done; i < len(in); i++ {
out[i] ^= mt[in[i]]
}
}
}

164
vendor/github.com/klauspost/reedsolomon/galois_amd64.s generated vendored Normal file
View File

@@ -0,0 +1,164 @@
//+build !noasm !appengine
// Copyright 2015, Klaus Post, see LICENSE for details.
// Based on http://www.snia.org/sites/default/files2/SDC2013/presentations/NewThinking/EthanMiller_Screaming_Fast_Galois_Field%20Arithmetic_SIMD%20Instructions.pdf
// and http://jerasure.org/jerasure/gf-complete/tree/master
// func galMulSSSE3Xor(low, high, in, out []byte)
TEXT ·galMulSSSE3Xor(SB), 7, $0
MOVQ low+0(FP), SI // SI: &low
MOVQ high+24(FP), DX // DX: &high
MOVOU (SI), X6 // X6 low
MOVOU (DX), X7 // X7: high
MOVQ $15, BX // BX: low mask
MOVQ BX, X8
PXOR X5, X5
MOVQ in+48(FP), SI // R11: &in
MOVQ in_len+56(FP), R9 // R9: len(in)
MOVQ out+72(FP), DX // DX: &out
PSHUFB X5, X8 // X8: lomask (unpacked)
SHRQ $4, R9 // len(in) / 16
CMPQ R9, $0
JEQ done_xor
loopback_xor:
MOVOU (SI), X0 // in[x]
MOVOU (DX), X4 // out[x]
MOVOU X0, X1 // in[x]
MOVOU X6, X2 // low copy
MOVOU X7, X3 // high copy
PSRLQ $4, X1 // X1: high input
PAND X8, X0 // X0: low input
PAND X8, X1 // X0: high input
PSHUFB X0, X2 // X2: mul low part
PSHUFB X1, X3 // X3: mul high part
PXOR X2, X3 // X3: Result
PXOR X4, X3 // X3: Result xor existing out
MOVOU X3, (DX) // Store
ADDQ $16, SI // in+=16
ADDQ $16, DX // out+=16
SUBQ $1, R9
JNZ loopback_xor
done_xor:
RET
// func galMulSSSE3(low, high, in, out []byte)
TEXT ·galMulSSSE3(SB), 7, $0
MOVQ low+0(FP), SI // SI: &low
MOVQ high+24(FP), DX // DX: &high
MOVOU (SI), X6 // X6 low
MOVOU (DX), X7 // X7: high
MOVQ $15, BX // BX: low mask
MOVQ BX, X8
PXOR X5, X5
MOVQ in+48(FP), SI // R11: &in
MOVQ in_len+56(FP), R9 // R9: len(in)
MOVQ out+72(FP), DX // DX: &out
PSHUFB X5, X8 // X8: lomask (unpacked)
SHRQ $4, R9 // len(in) / 16
CMPQ R9, $0
JEQ done
loopback:
MOVOU (SI), X0 // in[x]
MOVOU X0, X1 // in[x]
MOVOU X6, X2 // low copy
MOVOU X7, X3 // high copy
PSRLQ $4, X1 // X1: high input
PAND X8, X0 // X0: low input
PAND X8, X1 // X0: high input
PSHUFB X0, X2 // X2: mul low part
PSHUFB X1, X3 // X3: mul high part
PXOR X2, X3 // X3: Result
MOVOU X3, (DX) // Store
ADDQ $16, SI // in+=16
ADDQ $16, DX // out+=16
SUBQ $1, R9
JNZ loopback
done:
RET
// func galMulAVX2Xor(low, high, in, out []byte)
TEXT ·galMulAVX2Xor(SB), 7, $0
MOVQ low+0(FP), SI // SI: &low
MOVQ high+24(FP), DX // DX: &high
MOVQ $15, BX // BX: low mask
MOVQ BX, X5
MOVOU (SI), X6 // X6 low
MOVOU (DX), X7 // X7: high
MOVQ in_len+56(FP), R9 // R9: len(in)
LONG $0x384de3c4; WORD $0x01f6 // VINSERTI128 YMM6, YMM6, XMM6, 1 ; low
LONG $0x3845e3c4; WORD $0x01ff // VINSERTI128 YMM7, YMM7, XMM7, 1 ; high
LONG $0x787d62c4; BYTE $0xc5 // VPBROADCASTB YMM8, XMM5 ; X8: lomask (unpacked)
SHRQ $5, R9 // len(in) /32
MOVQ out+72(FP), DX // DX: &out
MOVQ in+48(FP), SI // R11: &in
TESTQ R9, R9
JZ done_xor_avx2
loopback_xor_avx2:
LONG $0x066ffec5 // VMOVDQU YMM0, [rsi]
LONG $0x226ffec5 // VMOVDQU YMM4, [rdx]
LONG $0xd073f5c5; BYTE $0x04 // VPSRLQ YMM1, YMM0, 4 ; X1: high input
LONG $0xdb7dc1c4; BYTE $0xc0 // VPAND YMM0, YMM0, YMM8 ; X0: low input
LONG $0xdb75c1c4; BYTE $0xc8 // VPAND YMM1, YMM1, YMM8 ; X1: high input
LONG $0x004de2c4; BYTE $0xd0 // VPSHUFB YMM2, YMM6, YMM0 ; X2: mul low part
LONG $0x0045e2c4; BYTE $0xd9 // VPSHUFB YMM3, YMM7, YMM1 ; X2: mul high part
LONG $0xdbefedc5 // VPXOR YMM3, YMM2, YMM3 ; X3: Result
LONG $0xe4efe5c5 // VPXOR YMM4, YMM3, YMM4 ; X4: Result
LONG $0x227ffec5 // VMOVDQU [rdx], YMM4
ADDQ $32, SI // in+=32
ADDQ $32, DX // out+=32
SUBQ $1, R9
JNZ loopback_xor_avx2
done_xor_avx2:
// VZEROUPPER
BYTE $0xc5; BYTE $0xf8; BYTE $0x77
RET
// func galMulAVX2(low, high, in, out []byte)
TEXT ·galMulAVX2(SB), 7, $0
MOVQ low+0(FP), SI // SI: &low
MOVQ high+24(FP), DX // DX: &high
MOVQ $15, BX // BX: low mask
MOVQ BX, X5
MOVOU (SI), X6 // X6 low
MOVOU (DX), X7 // X7: high
MOVQ in_len+56(FP), R9 // R9: len(in)
LONG $0x384de3c4; WORD $0x01f6 // VINSERTI128 YMM6, YMM6, XMM6, 1 ; low
LONG $0x3845e3c4; WORD $0x01ff // VINSERTI128 YMM7, YMM7, XMM7, 1 ; high
LONG $0x787d62c4; BYTE $0xc5 // VPBROADCASTB YMM8, XMM5 ; X8: lomask (unpacked)
SHRQ $5, R9 // len(in) /32
MOVQ out+72(FP), DX // DX: &out
MOVQ in+48(FP), SI // R11: &in
TESTQ R9, R9
JZ done_avx2
loopback_avx2:
LONG $0x066ffec5 // VMOVDQU YMM0, [rsi]
LONG $0xd073f5c5; BYTE $0x04 // VPSRLQ YMM1, YMM0, 4 ; X1: high input
LONG $0xdb7dc1c4; BYTE $0xc0 // VPAND YMM0, YMM0, YMM8 ; X0: low input
LONG $0xdb75c1c4; BYTE $0xc8 // VPAND YMM1, YMM1, YMM8 ; X1: high input
LONG $0x004de2c4; BYTE $0xd0 // VPSHUFB YMM2, YMM6, YMM0 ; X2: mul low part
LONG $0x0045e2c4; BYTE $0xd9 // VPSHUFB YMM3, YMM7, YMM1 ; X2: mul high part
LONG $0xe3efedc5 // VPXOR YMM4, YMM2, YMM3 ; X4: Result
LONG $0x227ffec5 // VMOVDQU [rdx], YMM4
ADDQ $32, SI // in+=32
ADDQ $32, DX // out+=32
SUBQ $1, R9
JNZ loopback_avx2
done_avx2:
BYTE $0xc5; BYTE $0xf8; BYTE $0x77 // VZEROUPPER
RET

View File

@@ -0,0 +1,19 @@
//+build !amd64 noasm appengine
// Copyright 2015, Klaus Post, see LICENSE for details.
package reedsolomon
func galMulSlice(c byte, in, out []byte, ssse3, avx2 bool) {
mt := mulTable[c]
for n, input := range in {
out[n] = mt[input]
}
}
func galMulSliceXor(c byte, in, out []byte, ssse3, avx2 bool) {
mt := mulTable[c]
for n, input := range in {
out[n] ^= mt[input]
}
}

132
vendor/github.com/klauspost/reedsolomon/gentables.go generated vendored Normal file
View File

@@ -0,0 +1,132 @@
//+build ignore
package main
import (
"fmt"
)
var logTable = [fieldSize]int16{
-1, 0, 1, 25, 2, 50, 26, 198,
3, 223, 51, 238, 27, 104, 199, 75,
4, 100, 224, 14, 52, 141, 239, 129,
28, 193, 105, 248, 200, 8, 76, 113,
5, 138, 101, 47, 225, 36, 15, 33,
53, 147, 142, 218, 240, 18, 130, 69,
29, 181, 194, 125, 106, 39, 249, 185,
201, 154, 9, 120, 77, 228, 114, 166,
6, 191, 139, 98, 102, 221, 48, 253,
226, 152, 37, 179, 16, 145, 34, 136,
54, 208, 148, 206, 143, 150, 219, 189,
241, 210, 19, 92, 131, 56, 70, 64,
30, 66, 182, 163, 195, 72, 126, 110,
107, 58, 40, 84, 250, 133, 186, 61,
202, 94, 155, 159, 10, 21, 121, 43,
78, 212, 229, 172, 115, 243, 167, 87,
7, 112, 192, 247, 140, 128, 99, 13,
103, 74, 222, 237, 49, 197, 254, 24,
227, 165, 153, 119, 38, 184, 180, 124,
17, 68, 146, 217, 35, 32, 137, 46,
55, 63, 209, 91, 149, 188, 207, 205,
144, 135, 151, 178, 220, 252, 190, 97,
242, 86, 211, 171, 20, 42, 93, 158,
132, 60, 57, 83, 71, 109, 65, 162,
31, 45, 67, 216, 183, 123, 164, 118,
196, 23, 73, 236, 127, 12, 111, 246,
108, 161, 59, 82, 41, 157, 85, 170,
251, 96, 134, 177, 187, 204, 62, 90,
203, 89, 95, 176, 156, 169, 160, 81,
11, 245, 22, 235, 122, 117, 44, 215,
79, 174, 213, 233, 230, 231, 173, 232,
116, 214, 244, 234, 168, 80, 88, 175,
}
const (
// The number of elements in the field.
fieldSize = 256
// The polynomial used to generate the logarithm table.
//
// There are a number of polynomials that work to generate
// a Galois field of 256 elements. The choice is arbitrary,
// and we just use the first one.
//
// The possibilities are: 29, 43, 45, 77, 95, 99, 101, 105,
//* 113, 135, 141, 169, 195, 207, 231, and 245.
generatingPolynomial = 29
)
func main() {
t := generateExpTable()
fmt.Printf("var expTable = %#v\n", t)
//t2 := generateMulTableSplit(t)
//fmt.Printf("var mulTable = %#v\n", t2)
low, high := generateMulTableHalf(t)
fmt.Printf("var mulTableLow = %#v\n", low)
fmt.Printf("var mulTableHigh = %#v\n", high)
}
/**
* Generates the inverse log table.
*/
func generateExpTable() []byte {
result := make([]byte, fieldSize*2-2)
for i := 1; i < fieldSize; i++ {
log := logTable[i]
result[log] = byte(i)
result[log+fieldSize-1] = byte(i)
}
return result
}
func generateMulTable(expTable []byte) []byte {
result := make([]byte, 256*256)
for v := range result {
a := byte(v & 0xff)
b := byte(v >> 8)
if a == 0 || b == 0 {
result[v] = 0
continue
}
logA := int(logTable[a])
logB := int(logTable[b])
result[v] = expTable[logA+logB]
}
return result
}
func generateMulTableSplit(expTable []byte) [256][256]byte {
var result [256][256]byte
for a := range result {
for b := range result[a] {
if a == 0 || b == 0 {
result[a][b] = 0
continue
}
logA := int(logTable[a])
logB := int(logTable[b])
result[a][b] = expTable[logA+logB]
}
}
return result
}
func generateMulTableHalf(expTable []byte) (low [256][16]byte, high [256][16]byte) {
for a := range low {
for b := range low {
result := 0
if !(a == 0 || b == 0) {
logA := int(logTable[a])
logB := int(logTable[b])
result = int(expTable[logA+logB])
}
if (b & 0xf) == b {
low[a][b] = byte(result)
}
if (b & 0xf0) == b {
high[a][b>>4] = byte(result)
}
}
}
return
}

View File

@@ -0,0 +1,160 @@
/**
* A thread-safe tree which caches inverted matrices.
*
* Copyright 2016, Peter Collins
*/
package reedsolomon
import (
"errors"
"sync"
)
// The tree uses a Reader-Writer mutex to make it thread-safe
// when accessing cached matrices and inserting new ones.
type inversionTree struct {
mutex *sync.RWMutex
root inversionNode
}
type inversionNode struct {
matrix matrix
children []*inversionNode
}
// newInversionTree initializes a tree for storing inverted matrices.
// Note that the root node is the identity matrix as it implies
// there were no errors with the original data.
func newInversionTree(dataShards, parityShards int) inversionTree {
identity, _ := identityMatrix(dataShards)
root := inversionNode{
matrix: identity,
children: make([]*inversionNode, dataShards+parityShards),
}
return inversionTree{
mutex: &sync.RWMutex{},
root: root,
}
}
// GetInvertedMatrix returns the cached inverted matrix or nil if it
// is not found in the tree keyed on the indices of invalid rows.
func (t inversionTree) GetInvertedMatrix(invalidIndices []int) matrix {
// Lock the tree for reading before accessing the tree.
t.mutex.RLock()
defer t.mutex.RUnlock()
// If no invalid indices were give we should return the root
// identity matrix.
if len(invalidIndices) == 0 {
return t.root.matrix
}
// Recursively search for the inverted matrix in the tree, passing in
// 0 as the parent index as we start at the root of the tree.
return t.root.getInvertedMatrix(invalidIndices, 0)
}
// errAlreadySet is returned if the root node matrix is overwritten
var errAlreadySet = errors.New("the root node identity matrix is already set")
// InsertInvertedMatrix inserts a new inverted matrix into the tree
// keyed by the indices of invalid rows. The total number of shards
// is required for creating the proper length lists of child nodes for
// each node.
func (t inversionTree) InsertInvertedMatrix(invalidIndices []int, matrix matrix, shards int) error {
// If no invalid indices were given then we are done because the
// root node is already set with the identity matrix.
if len(invalidIndices) == 0 {
return errAlreadySet
}
if !matrix.IsSquare() {
return errNotSquare
}
// Lock the tree for writing and reading before accessing the tree.
t.mutex.Lock()
defer t.mutex.Unlock()
// Recursively create nodes for the inverted matrix in the tree until
// we reach the node to insert the matrix to. We start by passing in
// 0 as the parent index as we start at the root of the tree.
t.root.insertInvertedMatrix(invalidIndices, matrix, shards, 0)
return nil
}
func (n inversionNode) getInvertedMatrix(invalidIndices []int, parent int) matrix {
// Get the child node to search next from the list of children. The
// list of children starts relative to the parent index passed in
// because the indices of invalid rows is sorted (by default). As we
// search recursively, the first invalid index gets popped off the list,
// so when searching through the list of children, use that first invalid
// index to find the child node.
firstIndex := invalidIndices[0]
node := n.children[firstIndex-parent]
// If the child node doesn't exist in the list yet, fail fast by
// returning, so we can construct and insert the proper inverted matrix.
if node == nil {
return nil
}
// If there's more than one invalid index left in the list we should
// keep searching recursively.
if len(invalidIndices) > 1 {
// Search recursively on the child node by passing in the invalid indices
// with the first index popped off the front. Also the parent index to
// pass down is the first index plus one.
return node.getInvertedMatrix(invalidIndices[1:], firstIndex+1)
}
// If there aren't any more invalid indices to search, we've found our
// node. Return it, however keep in mind that the matrix could still be
// nil because intermediary nodes in the tree are created sometimes with
// their inversion matrices uninitialized.
return node.matrix
}
func (n inversionNode) insertInvertedMatrix(invalidIndices []int, matrix matrix, shards, parent int) {
// As above, get the child node to search next from the list of children.
// The list of children starts relative to the parent index passed in
// because the indices of invalid rows is sorted (by default). As we
// search recursively, the first invalid index gets popped off the list,
// so when searching through the list of children, use that first invalid
// index to find the child node.
firstIndex := invalidIndices[0]
node := n.children[firstIndex-parent]
// If the child node doesn't exist in the list yet, create a new
// node because we have the writer lock and add it to the list
// of children.
if node == nil {
// Make the length of the list of children equal to the number
// of shards minus the first invalid index because the list of
// invalid indices is sorted, so only this length of errors
// are possible in the tree.
node = &inversionNode{
children: make([]*inversionNode, shards-firstIndex),
}
// Insert the new node into the tree at the first index relative
// to the parent index that was given in this recursive call.
n.children[firstIndex-parent] = node
}
// If there's more than one invalid index left in the list we should
// keep searching recursively in order to find the node to add our
// matrix.
if len(invalidIndices) > 1 {
// As above, search recursively on the child node by passing in
// the invalid indices with the first index popped off the front.
// Also the total number of shards and parent index are passed down
// which is equal to the first index plus one.
node.insertInvertedMatrix(invalidIndices[1:], matrix, shards, firstIndex+1)
} else {
// If there aren't any more invalid indices to search, we've found our
// node. Cache the inverted matrix in this node.
node.matrix = matrix
}
}

279
vendor/github.com/klauspost/reedsolomon/matrix.go generated vendored Normal file
View File

@@ -0,0 +1,279 @@
/**
* Matrix Algebra over an 8-bit Galois Field
*
* Copyright 2015, Klaus Post
* Copyright 2015, Backblaze, Inc.
*/
package reedsolomon
import (
"errors"
"fmt"
"strconv"
"strings"
)
// byte[row][col]
type matrix [][]byte
// newMatrix returns a matrix of zeros.
func newMatrix(rows, cols int) (matrix, error) {
if rows <= 0 {
return nil, errInvalidRowSize
}
if cols <= 0 {
return nil, errInvalidColSize
}
m := matrix(make([][]byte, rows))
for i := range m {
m[i] = make([]byte, cols)
}
return m, nil
}
// NewMatrixData initializes a matrix with the given row-major data.
// Note that data is not copied from input.
func newMatrixData(data [][]byte) (matrix, error) {
m := matrix(data)
err := m.Check()
if err != nil {
return nil, err
}
return m, nil
}
// IdentityMatrix returns an identity matrix of the given size.
func identityMatrix(size int) (matrix, error) {
m, err := newMatrix(size, size)
if err != nil {
return nil, err
}
for i := range m {
m[i][i] = 1
}
return m, nil
}
// errInvalidRowSize will be returned if attempting to create a matrix with negative or zero row number.
var errInvalidRowSize = errors.New("invalid row size")
// errInvalidColSize will be returned if attempting to create a matrix with negative or zero column number.
var errInvalidColSize = errors.New("invalid column size")
// errColSizeMismatch is returned if the size of matrix columns mismatch.
var errColSizeMismatch = errors.New("column size is not the same for all rows")
func (m matrix) Check() error {
rows := len(m)
if rows <= 0 {
return errInvalidRowSize
}
cols := len(m[0])
if cols <= 0 {
return errInvalidColSize
}
for _, col := range m {
if len(col) != cols {
return errColSizeMismatch
}
}
return nil
}
// String returns a human-readable string of the matrix contents.
//
// Example: [[1, 2], [3, 4]]
func (m matrix) String() string {
rowOut := make([]string, 0, len(m))
for _, row := range m {
colOut := make([]string, 0, len(row))
for _, col := range row {
colOut = append(colOut, strconv.Itoa(int(col)))
}
rowOut = append(rowOut, "["+strings.Join(colOut, ", ")+"]")
}
return "[" + strings.Join(rowOut, ", ") + "]"
}
// Multiply multiplies this matrix (the one on the left) by another
// matrix (the one on the right) and returns a new matrix with the result.
func (m matrix) Multiply(right matrix) (matrix, error) {
if len(m[0]) != len(right) {
return nil, fmt.Errorf("columns on left (%d) is different than rows on right (%d)", len(m[0]), len(right))
}
result, _ := newMatrix(len(m), len(right[0]))
for r, row := range result {
for c := range row {
var value byte
for i := range m[0] {
value ^= galMultiply(m[r][i], right[i][c])
}
result[r][c] = value
}
}
return result, nil
}
// Augment returns the concatenation of this matrix and the matrix on the right.
func (m matrix) Augment(right matrix) (matrix, error) {
if len(m) != len(right) {
return nil, errMatrixSize
}
result, _ := newMatrix(len(m), len(m[0])+len(right[0]))
for r, row := range m {
for c := range row {
result[r][c] = m[r][c]
}
cols := len(m[0])
for c := range right[0] {
result[r][cols+c] = right[r][c]
}
}
return result, nil
}
// errMatrixSize is returned if matrix dimensions are doesn't match.
var errMatrixSize = errors.New("matrix sizes does not match")
func (m matrix) SameSize(n matrix) error {
if len(m) != len(n) {
return errMatrixSize
}
for i := range m {
if len(m[i]) != len(n[i]) {
return errMatrixSize
}
}
return nil
}
// Returns a part of this matrix. Data is copied.
func (m matrix) SubMatrix(rmin, cmin, rmax, cmax int) (matrix, error) {
result, err := newMatrix(rmax-rmin, cmax-cmin)
if err != nil {
return nil, err
}
// OPTME: If used heavily, use copy function to copy slice
for r := rmin; r < rmax; r++ {
for c := cmin; c < cmax; c++ {
result[r-rmin][c-cmin] = m[r][c]
}
}
return result, nil
}
// SwapRows Exchanges two rows in the matrix.
func (m matrix) SwapRows(r1, r2 int) error {
if r1 < 0 || len(m) <= r1 || r2 < 0 || len(m) <= r2 {
return errInvalidRowSize
}
m[r2], m[r1] = m[r1], m[r2]
return nil
}
// IsSquare will return true if the matrix is square
// and nil if the matrix is square
func (m matrix) IsSquare() bool {
return len(m) == len(m[0])
}
// errSingular is returned if the matrix is singular and cannot be inversed
var errSingular = errors.New("matrix is singular")
// errNotSquare is returned if attempting to inverse a non-square matrix.
var errNotSquare = errors.New("only square matrices can be inverted")
// Invert returns the inverse of this matrix.
// Returns ErrSingular when the matrix is singular and doesn't have an inverse.
// The matrix must be square, otherwise ErrNotSquare is returned.
func (m matrix) Invert() (matrix, error) {
if !m.IsSquare() {
return nil, errNotSquare
}
size := len(m)
work, _ := identityMatrix(size)
work, _ = m.Augment(work)
err := work.gaussianElimination()
if err != nil {
return nil, err
}
return work.SubMatrix(0, size, size, size*2)
}
func (m matrix) gaussianElimination() error {
rows := len(m)
columns := len(m[0])
// Clear out the part below the main diagonal and scale the main
// diagonal to be 1.
for r := 0; r < rows; r++ {
// If the element on the diagonal is 0, find a row below
// that has a non-zero and swap them.
if m[r][r] == 0 {
for rowBelow := r + 1; rowBelow < rows; rowBelow++ {
if m[rowBelow][r] != 0 {
m.SwapRows(r, rowBelow)
break
}
}
}
// If we couldn't find one, the matrix is singular.
if m[r][r] == 0 {
return errSingular
}
// Scale to 1.
if m[r][r] != 1 {
scale := galDivide(1, m[r][r])
for c := 0; c < columns; c++ {
m[r][c] = galMultiply(m[r][c], scale)
}
}
// Make everything below the 1 be a 0 by subtracting
// a multiple of it. (Subtraction and addition are
// both exclusive or in the Galois field.)
for rowBelow := r + 1; rowBelow < rows; rowBelow++ {
if m[rowBelow][r] != 0 {
scale := m[rowBelow][r]
for c := 0; c < columns; c++ {
m[rowBelow][c] ^= galMultiply(scale, m[r][c])
}
}
}
}
// Now clear the part above the main diagonal.
for d := 0; d < rows; d++ {
for rowAbove := 0; rowAbove < d; rowAbove++ {
if m[rowAbove][d] != 0 {
scale := m[rowAbove][d]
for c := 0; c < columns; c++ {
m[rowAbove][c] ^= galMultiply(scale, m[d][c])
}
}
}
}
return nil
}
// Create a Vandermonde matrix, which is guaranteed to have the
// property that any subset of rows that forms a square matrix
// is invertible.
func vandermonde(rows, cols int) (matrix, error) {
result, err := newMatrix(rows, cols)
if err != nil {
return nil, err
}
for r, row := range result {
for c := range row {
result[r][c] = galExp(byte(r), c)
}
}
return result, nil
}

67
vendor/github.com/klauspost/reedsolomon/options.go generated vendored Normal file
View File

@@ -0,0 +1,67 @@
package reedsolomon
import (
"runtime"
"github.com/klauspost/cpuid"
)
// Option allows to override processing parameters.
type Option func(*options)
type options struct {
maxGoroutines int
minSplitSize int
useAVX2, useSSSE3 bool
}
var defaultOptions = options{
maxGoroutines: 50,
minSplitSize: 512,
}
func init() {
if runtime.GOMAXPROCS(0) <= 1 {
defaultOptions.maxGoroutines = 1
}
// Detect CPU capabilities.
defaultOptions.useSSSE3 = cpuid.CPU.SSSE3()
defaultOptions.useAVX2 = cpuid.CPU.AVX2()
}
// WithMaxGoroutines is the maximum number of goroutines number for encoding & decoding.
// Jobs will be split into this many parts, unless each goroutine would have to process
// less than minSplitSize bytes (set with WithMinSplitSize).
// For the best speed, keep this well above the GOMAXPROCS number for more fine grained
// scheduling.
// If n <= 0, it is ignored.
func WithMaxGoroutines(n int) Option {
return func(o *options) {
if n > 0 {
o.maxGoroutines = n
}
}
}
// MinSplitSize Is the minimum encoding size in bytes per goroutine.
// See WithMaxGoroutines on how jobs are split.
// If n <= 0, it is ignored.
func WithMinSplitSize(n int) Option {
return func(o *options) {
if n > 0 {
o.minSplitSize = n
}
}
}
func withSSE3(enabled bool) Option {
return func(o *options) {
o.useSSSE3 = enabled
}
}
func withAVX2(enabled bool) Option {
return func(o *options) {
o.useAVX2 = enabled
}
}

596
vendor/github.com/klauspost/reedsolomon/reedsolomon.go generated vendored Normal file
View File

@@ -0,0 +1,596 @@
/**
* Reed-Solomon Coding over 8-bit values.
*
* Copyright 2015, Klaus Post
* Copyright 2015, Backblaze, Inc.
*/
// Package reedsolomon enables Erasure Coding in Go
//
// For usage and examples, see https://github.com/klauspost/reedsolomon
//
package reedsolomon
import (
"bytes"
"errors"
"io"
"sync"
)
// Encoder is an interface to encode Reed-Salomon parity sets for your data.
type Encoder interface {
// Encodes parity for a set of data shards.
// Input is 'shards' containing data shards followed by parity shards.
// The number of shards must match the number given to New().
// Each shard is a byte array, and they must all be the same size.
// The parity shards will always be overwritten and the data shards
// will remain the same, so it is safe for you to read from the
// data shards while this is running.
Encode(shards [][]byte) error
// Verify returns true if the parity shards contain correct data.
// The data is the same format as Encode. No data is modified, so
// you are allowed to read from data while this is running.
Verify(shards [][]byte) (bool, error)
// Reconstruct will recreate the missing shards if possible.
//
// Given a list of shards, some of which contain data, fills in the
// ones that don't have data.
//
// The length of the array must be equal to the total number of shards.
// You indicate that a shard is missing by setting it to nil.
//
// If there are too few shards to reconstruct the missing
// ones, ErrTooFewShards will be returned.
//
// The reconstructed shard set is complete, but integrity is not verified.
// Use the Verify function to check if data set is ok.
Reconstruct(shards [][]byte) error
// Split a data slice into the number of shards given to the encoder,
// and create empty parity shards.
//
// The data will be split into equally sized shards.
// If the data size isn't dividable by the number of shards,
// the last shard will contain extra zeros.
//
// There must be at least 1 byte otherwise ErrShortData will be
// returned.
//
// The data will not be copied, except for the last shard, so you
// should not modify the data of the input slice afterwards.
Split(data []byte) ([][]byte, error)
// Join the shards and write the data segment to dst.
//
// Only the data shards are considered.
// You must supply the exact output size you want.
// If there are to few shards given, ErrTooFewShards will be returned.
// If the total data size is less than outSize, ErrShortData will be returned.
Join(dst io.Writer, shards [][]byte, outSize int) error
}
// reedSolomon contains a matrix for a specific
// distribution of datashards and parity shards.
// Construct if using New()
type reedSolomon struct {
DataShards int // Number of data shards, should not be modified.
ParityShards int // Number of parity shards, should not be modified.
Shards int // Total number of shards. Calculated, and should not be modified.
m matrix
tree inversionTree
parity [][]byte
o options
}
// ErrInvShardNum will be returned by New, if you attempt to create
// an Encoder where either data or parity shards is zero or less.
var ErrInvShardNum = errors.New("cannot create Encoder with zero or less data/parity shards")
// ErrMaxShardNum will be returned by New, if you attempt to create
// an Encoder where data and parity shards cannot be bigger than
// Galois field GF(2^8) - 1.
var ErrMaxShardNum = errors.New("cannot create Encoder with 255 or more data+parity shards")
// New creates a new encoder and initializes it to
// the number of data shards and parity shards that
// you want to use. You can reuse this encoder.
// Note that the maximum number of data shards is 256.
// If no options are supplied, default options are used.
func New(dataShards, parityShards int, opts ...Option) (Encoder, error) {
r := reedSolomon{
DataShards: dataShards,
ParityShards: parityShards,
Shards: dataShards + parityShards,
o: defaultOptions,
}
for _, opt := range opts {
opt(&r.o)
}
if dataShards <= 0 || parityShards <= 0 {
return nil, ErrInvShardNum
}
if dataShards+parityShards > 255 {
return nil, ErrMaxShardNum
}
// Start with a Vandermonde matrix. This matrix would work,
// in theory, but doesn't have the property that the data
// shards are unchanged after encoding.
vm, err := vandermonde(r.Shards, dataShards)
if err != nil {
return nil, err
}
// Multiply by the inverse of the top square of the matrix.
// This will make the top square be the identity matrix, but
// preserve the property that any square subset of rows is
// invertible.
top, _ := vm.SubMatrix(0, 0, dataShards, dataShards)
top, _ = top.Invert()
r.m, _ = vm.Multiply(top)
// Inverted matrices are cached in a tree keyed by the indices
// of the invalid rows of the data to reconstruct.
// The inversion root node will have the identity matrix as
// its inversion matrix because it implies there are no errors
// with the original data.
r.tree = newInversionTree(dataShards, parityShards)
r.parity = make([][]byte, parityShards)
for i := range r.parity {
r.parity[i] = r.m[dataShards+i]
}
return &r, err
}
// ErrTooFewShards is returned if too few shards where given to
// Encode/Verify/Reconstruct. It will also be returned from Reconstruct
// if there were too few shards to reconstruct the missing data.
var ErrTooFewShards = errors.New("too few shards given")
// Encodes parity for a set of data shards.
// An array 'shards' containing data shards followed by parity shards.
// The number of shards must match the number given to New.
// Each shard is a byte array, and they must all be the same size.
// The parity shards will always be overwritten and the data shards
// will remain the same.
func (r reedSolomon) Encode(shards [][]byte) error {
if len(shards) != r.Shards {
return ErrTooFewShards
}
err := checkShards(shards, false)
if err != nil {
return err
}
// Get the slice of output buffers.
output := shards[r.DataShards:]
// Do the coding.
r.codeSomeShards(r.parity, shards[0:r.DataShards], output, r.ParityShards, len(shards[0]))
return nil
}
// Verify returns true if the parity shards contain the right data.
// The data is the same format as Encode. No data is modified.
func (r reedSolomon) Verify(shards [][]byte) (bool, error) {
if len(shards) != r.Shards {
return false, ErrTooFewShards
}
err := checkShards(shards, false)
if err != nil {
return false, err
}
// Slice of buffers being checked.
toCheck := shards[r.DataShards:]
// Do the checking.
return r.checkSomeShards(r.parity, shards[0:r.DataShards], toCheck, r.ParityShards, len(shards[0])), nil
}
// Multiplies a subset of rows from a coding matrix by a full set of
// input shards to produce some output shards.
// 'matrixRows' is The rows from the matrix to use.
// 'inputs' An array of byte arrays, each of which is one input shard.
// The number of inputs used is determined by the length of each matrix row.
// outputs Byte arrays where the computed shards are stored.
// The number of outputs computed, and the
// number of matrix rows used, is determined by
// outputCount, which is the number of outputs to compute.
func (r reedSolomon) codeSomeShards(matrixRows, inputs, outputs [][]byte, outputCount, byteCount int) {
if r.o.maxGoroutines > 1 && byteCount > r.o.minSplitSize {
r.codeSomeShardsP(matrixRows, inputs, outputs, outputCount, byteCount)
return
}
for c := 0; c < r.DataShards; c++ {
in := inputs[c]
for iRow := 0; iRow < outputCount; iRow++ {
if c == 0 {
galMulSlice(matrixRows[iRow][c], in, outputs[iRow], r.o.useSSSE3, r.o.useAVX2)
} else {
galMulSliceXor(matrixRows[iRow][c], in, outputs[iRow], r.o.useSSSE3, r.o.useAVX2)
}
}
}
}
// Perform the same as codeSomeShards, but split the workload into
// several goroutines.
func (r reedSolomon) codeSomeShardsP(matrixRows, inputs, outputs [][]byte, outputCount, byteCount int) {
var wg sync.WaitGroup
do := byteCount / r.o.maxGoroutines
if do < r.o.minSplitSize {
do = r.o.minSplitSize
}
start := 0
for start < byteCount {
if start+do > byteCount {
do = byteCount - start
}
wg.Add(1)
go func(start, stop int) {
for c := 0; c < r.DataShards; c++ {
in := inputs[c]
for iRow := 0; iRow < outputCount; iRow++ {
if c == 0 {
galMulSlice(matrixRows[iRow][c], in[start:stop], outputs[iRow][start:stop], r.o.useSSSE3, r.o.useAVX2)
} else {
galMulSliceXor(matrixRows[iRow][c], in[start:stop], outputs[iRow][start:stop], r.o.useSSSE3, r.o.useAVX2)
}
}
}
wg.Done()
}(start, start+do)
start += do
}
wg.Wait()
}
// checkSomeShards is mostly the same as codeSomeShards,
// except this will check values and return
// as soon as a difference is found.
func (r reedSolomon) checkSomeShards(matrixRows, inputs, toCheck [][]byte, outputCount, byteCount int) bool {
if r.o.maxGoroutines > 1 && byteCount > r.o.minSplitSize {
return r.checkSomeShardsP(matrixRows, inputs, toCheck, outputCount, byteCount)
}
outputs := make([][]byte, len(toCheck))
for i := range outputs {
outputs[i] = make([]byte, byteCount)
}
for c := 0; c < r.DataShards; c++ {
in := inputs[c]
for iRow := 0; iRow < outputCount; iRow++ {
galMulSliceXor(matrixRows[iRow][c], in, outputs[iRow], r.o.useSSSE3, r.o.useAVX2)
}
}
for i, calc := range outputs {
if !bytes.Equal(calc, toCheck[i]) {
return false
}
}
return true
}
func (r reedSolomon) checkSomeShardsP(matrixRows, inputs, toCheck [][]byte, outputCount, byteCount int) bool {
same := true
var mu sync.RWMutex // For above
var wg sync.WaitGroup
do := byteCount / r.o.maxGoroutines
if do < r.o.minSplitSize {
do = r.o.minSplitSize
}
start := 0
for start < byteCount {
if start+do > byteCount {
do = byteCount - start
}
wg.Add(1)
go func(start, do int) {
defer wg.Done()
outputs := make([][]byte, len(toCheck))
for i := range outputs {
outputs[i] = make([]byte, do)
}
for c := 0; c < r.DataShards; c++ {
mu.RLock()
if !same {
mu.RUnlock()
return
}
mu.RUnlock()
in := inputs[c][start : start+do]
for iRow := 0; iRow < outputCount; iRow++ {
galMulSliceXor(matrixRows[iRow][c], in, outputs[iRow], r.o.useSSSE3, r.o.useAVX2)
}
}
for i, calc := range outputs {
if !bytes.Equal(calc, toCheck[i][start:start+do]) {
mu.Lock()
same = false
mu.Unlock()
return
}
}
}(start, do)
start += do
}
wg.Wait()
return same
}
// ErrShardNoData will be returned if there are no shards,
// or if the length of all shards is zero.
var ErrShardNoData = errors.New("no shard data")
// ErrShardSize is returned if shard length isn't the same for all
// shards.
var ErrShardSize = errors.New("shard sizes does not match")
// checkShards will check if shards are the same size
// or 0, if allowed. An error is returned if this fails.
// An error is also returned if all shards are size 0.
func checkShards(shards [][]byte, nilok bool) error {
size := shardSize(shards)
if size == 0 {
return ErrShardNoData
}
for _, shard := range shards {
if len(shard) != size {
if len(shard) != 0 || !nilok {
return ErrShardSize
}
}
}
return nil
}
// shardSize return the size of a single shard.
// The first non-zero size is returned,
// or 0 if all shards are size 0.
func shardSize(shards [][]byte) int {
for _, shard := range shards {
if len(shard) != 0 {
return len(shard)
}
}
return 0
}
// Reconstruct will recreate the missing shards, if possible.
//
// Given a list of shards, some of which contain data, fills in the
// ones that don't have data.
//
// The length of the array must be equal to Shards.
// You indicate that a shard is missing by setting it to nil.
//
// If there are too few shards to reconstruct the missing
// ones, ErrTooFewShards will be returned.
//
// The reconstructed shard set is complete, but integrity is not verified.
// Use the Verify function to check if data set is ok.
func (r reedSolomon) Reconstruct(shards [][]byte) error {
if len(shards) != r.Shards {
return ErrTooFewShards
}
// Check arguments.
err := checkShards(shards, true)
if err != nil {
return err
}
shardSize := shardSize(shards)
// Quick check: are all of the shards present? If so, there's
// nothing to do.
numberPresent := 0
for i := 0; i < r.Shards; i++ {
if len(shards[i]) != 0 {
numberPresent++
}
}
if numberPresent == r.Shards {
// Cool. All of the shards data data. We don't
// need to do anything.
return nil
}
// More complete sanity check
if numberPresent < r.DataShards {
return ErrTooFewShards
}
// Pull out an array holding just the shards that
// correspond to the rows of the submatrix. These shards
// will be the input to the decoding process that re-creates
// the missing data shards.
//
// Also, create an array of indices of the valid rows we do have
// and the invalid rows we don't have up until we have enough valid rows.
subShards := make([][]byte, r.DataShards)
validIndices := make([]int, r.DataShards)
invalidIndices := make([]int, 0)
subMatrixRow := 0
for matrixRow := 0; matrixRow < r.Shards && subMatrixRow < r.DataShards; matrixRow++ {
if len(shards[matrixRow]) != 0 {
subShards[subMatrixRow] = shards[matrixRow]
validIndices[subMatrixRow] = matrixRow
subMatrixRow++
} else {
invalidIndices = append(invalidIndices, matrixRow)
}
}
// Attempt to get the cached inverted matrix out of the tree
// based on the indices of the invalid rows.
dataDecodeMatrix := r.tree.GetInvertedMatrix(invalidIndices)
// If the inverted matrix isn't cached in the tree yet we must
// construct it ourselves and insert it into the tree for the
// future. In this way the inversion tree is lazily loaded.
if dataDecodeMatrix == nil {
// Pull out the rows of the matrix that correspond to the
// shards that we have and build a square matrix. This
// matrix could be used to generate the shards that we have
// from the original data.
subMatrix, _ := newMatrix(r.DataShards, r.DataShards)
for subMatrixRow, validIndex := range validIndices {
for c := 0; c < r.DataShards; c++ {
subMatrix[subMatrixRow][c] = r.m[validIndex][c]
}
}
// Invert the matrix, so we can go from the encoded shards
// back to the original data. Then pull out the row that
// generates the shard that we want to decode. Note that
// since this matrix maps back to the original data, it can
// be used to create a data shard, but not a parity shard.
dataDecodeMatrix, err = subMatrix.Invert()
if err != nil {
return err
}
// Cache the inverted matrix in the tree for future use keyed on the
// indices of the invalid rows.
err = r.tree.InsertInvertedMatrix(invalidIndices, dataDecodeMatrix, r.Shards)
if err != nil {
return err
}
}
// Re-create any data shards that were missing.
//
// The input to the coding is all of the shards we actually
// have, and the output is the missing data shards. The computation
// is done using the special decode matrix we just built.
outputs := make([][]byte, r.ParityShards)
matrixRows := make([][]byte, r.ParityShards)
outputCount := 0
for iShard := 0; iShard < r.DataShards; iShard++ {
if len(shards[iShard]) == 0 {
shards[iShard] = make([]byte, shardSize)
outputs[outputCount] = shards[iShard]
matrixRows[outputCount] = dataDecodeMatrix[iShard]
outputCount++
}
}
r.codeSomeShards(matrixRows, subShards, outputs[:outputCount], outputCount, shardSize)
// Now that we have all of the data shards intact, we can
// compute any of the parity that is missing.
//
// The input to the coding is ALL of the data shards, including
// any that we just calculated. The output is whichever of the
// data shards were missing.
outputCount = 0
for iShard := r.DataShards; iShard < r.Shards; iShard++ {
if len(shards[iShard]) == 0 {
shards[iShard] = make([]byte, shardSize)
outputs[outputCount] = shards[iShard]
matrixRows[outputCount] = r.parity[iShard-r.DataShards]
outputCount++
}
}
r.codeSomeShards(matrixRows, shards[:r.DataShards], outputs[:outputCount], outputCount, shardSize)
return nil
}
// ErrShortData will be returned by Split(), if there isn't enough data
// to fill the number of shards.
var ErrShortData = errors.New("not enough data to fill the number of requested shards")
// Split a data slice into the number of shards given to the encoder,
// and create empty parity shards.
//
// The data will be split into equally sized shards.
// If the data size isn't divisible by the number of shards,
// the last shard will contain extra zeros.
//
// There must be at least 1 byte otherwise ErrShortData will be
// returned.
//
// The data will not be copied, except for the last shard, so you
// should not modify the data of the input slice afterwards.
func (r reedSolomon) Split(data []byte) ([][]byte, error) {
if len(data) == 0 {
return nil, ErrShortData
}
// Calculate number of bytes per shard.
perShard := (len(data) + r.DataShards - 1) / r.DataShards
// Pad data to r.Shards*perShard.
padding := make([]byte, (r.Shards*perShard)-len(data))
data = append(data, padding...)
// Split into equal-length shards.
dst := make([][]byte, r.Shards)
for i := range dst {
dst[i] = data[:perShard]
data = data[perShard:]
}
return dst, nil
}
// ErrReconstructRequired is returned if too few data shards are intact and a
// reconstruction is required before you can successfully join the shards.
var ErrReconstructRequired = errors.New("reconstruction required as one or more required data shards are nil")
// Join the shards and write the data segment to dst.
//
// Only the data shards are considered.
// You must supply the exact output size you want.
//
// If there are to few shards given, ErrTooFewShards will be returned.
// If the total data size is less than outSize, ErrShortData will be returned.
// If one or more required data shards are nil, ErrReconstructRequired will be returned.
func (r reedSolomon) Join(dst io.Writer, shards [][]byte, outSize int) error {
// Do we have enough shards?
if len(shards) < r.DataShards {
return ErrTooFewShards
}
shards = shards[:r.DataShards]
// Do we have enough data?
size := 0
for _, shard := range shards {
if shard == nil {
return ErrReconstructRequired
}
size += len(shard)
// Do we have enough data already?
if size >= outSize {
break
}
}
if size < outSize {
return ErrShortData
}
// Copy data to dst
write := outSize
for _, shard := range shards {
if write < len(shard) {
_, err := dst.Write(shard[:write])
return err
}
n, err := dst.Write(shard)
if err != nil {
return err
}
write -= n
}
return nil
}

575
vendor/github.com/klauspost/reedsolomon/streaming.go generated vendored Normal file
View File

@@ -0,0 +1,575 @@
/**
* Reed-Solomon Coding over 8-bit values.
*
* Copyright 2015, Klaus Post
* Copyright 2015, Backblaze, Inc.
*/
package reedsolomon
import (
"bytes"
"errors"
"fmt"
"io"
"sync"
)
// StreamEncoder is an interface to encode Reed-Salomon parity sets for your data.
// It provides a fully streaming interface, and processes data in blocks of up to 4MB.
//
// For small shard sizes, 10MB and below, it is recommended to use the in-memory interface,
// since the streaming interface has a start up overhead.
//
// For all operations, no readers and writers should not assume any order/size of
// individual reads/writes.
//
// For usage examples, see "stream-encoder.go" and "streamdecoder.go" in the examples
// folder.
type StreamEncoder interface {
// Encodes parity shards for a set of data shards.
//
// Input is 'shards' containing readers for data shards followed by parity shards
// io.Writer.
//
// The number of shards must match the number given to NewStream().
//
// Each reader must supply the same number of bytes.
//
// The parity shards will be written to the writer.
// The number of bytes written will match the input size.
//
// If a data stream returns an error, a StreamReadError type error
// will be returned. If a parity writer returns an error, a
// StreamWriteError will be returned.
Encode(data []io.Reader, parity []io.Writer) error
// Verify returns true if the parity shards contain correct data.
//
// The number of shards must match the number total data+parity shards
// given to NewStream().
//
// Each reader must supply the same number of bytes.
// If a shard stream returns an error, a StreamReadError type error
// will be returned.
Verify(shards []io.Reader) (bool, error)
// Reconstruct will recreate the missing shards if possible.
//
// Given a list of valid shards (to read) and invalid shards (to write)
//
// You indicate that a shard is missing by setting it to nil in the 'valid'
// slice and at the same time setting a non-nil writer in "fill".
// An index cannot contain both non-nil 'valid' and 'fill' entry.
// If both are provided 'ErrReconstructMismatch' is returned.
//
// If there are too few shards to reconstruct the missing
// ones, ErrTooFewShards will be returned.
//
// The reconstructed shard set is complete, but integrity is not verified.
// Use the Verify function to check if data set is ok.
Reconstruct(valid []io.Reader, fill []io.Writer) error
// Split a an input stream into the number of shards given to the encoder.
//
// The data will be split into equally sized shards.
// If the data size isn't dividable by the number of shards,
// the last shard will contain extra zeros.
//
// You must supply the total size of your input.
// 'ErrShortData' will be returned if it is unable to retrieve the
// number of bytes indicated.
Split(data io.Reader, dst []io.Writer, size int64) (err error)
// Join the shards and write the data segment to dst.
//
// Only the data shards are considered.
//
// You must supply the exact output size you want.
// If there are to few shards given, ErrTooFewShards will be returned.
// If the total data size is less than outSize, ErrShortData will be returned.
Join(dst io.Writer, shards []io.Reader, outSize int64) error
}
// StreamReadError is returned when a read error is encountered
// that relates to a supplied stream.
// This will allow you to find out which reader has failed.
type StreamReadError struct {
Err error // The error
Stream int // The stream number on which the error occurred
}
// Error returns the error as a string
func (s StreamReadError) Error() string {
return fmt.Sprintf("error reading stream %d: %s", s.Stream, s.Err)
}
// String returns the error as a string
func (s StreamReadError) String() string {
return s.Error()
}
// StreamWriteError is returned when a write error is encountered
// that relates to a supplied stream. This will allow you to
// find out which reader has failed.
type StreamWriteError struct {
Err error // The error
Stream int // The stream number on which the error occurred
}
// Error returns the error as a string
func (s StreamWriteError) Error() string {
return fmt.Sprintf("error writing stream %d: %s", s.Stream, s.Err)
}
// String returns the error as a string
func (s StreamWriteError) String() string {
return s.Error()
}
// rsStream contains a matrix for a specific
// distribution of datashards and parity shards.
// Construct if using NewStream()
type rsStream struct {
r *reedSolomon
bs int // Block size
// Shard reader
readShards func(dst [][]byte, in []io.Reader) error
// Shard writer
writeShards func(out []io.Writer, in [][]byte) error
creads bool
cwrites bool
}
// NewStream creates a new encoder and initializes it to
// the number of data shards and parity shards that
// you want to use. You can reuse this encoder.
// Note that the maximum number of data shards is 256.
func NewStream(dataShards, parityShards int, o ...Option) (StreamEncoder, error) {
enc, err := New(dataShards, parityShards, o...)
if err != nil {
return nil, err
}
rs := enc.(*reedSolomon)
r := rsStream{r: rs, bs: 4 << 20}
r.readShards = readShards
r.writeShards = writeShards
return &r, err
}
// NewStreamC creates a new encoder and initializes it to
// the number of data shards and parity shards given.
//
// This functions as 'NewStream', but allows you to enable CONCURRENT reads and writes.
func NewStreamC(dataShards, parityShards int, conReads, conWrites bool, o ...Option) (StreamEncoder, error) {
enc, err := New(dataShards, parityShards, o...)
if err != nil {
return nil, err
}
rs := enc.(*reedSolomon)
r := rsStream{r: rs, bs: 4 << 20}
r.readShards = readShards
r.writeShards = writeShards
if conReads {
r.readShards = cReadShards
}
if conWrites {
r.writeShards = cWriteShards
}
return &r, err
}
func createSlice(n, length int) [][]byte {
out := make([][]byte, n)
for i := range out {
out[i] = make([]byte, length)
}
return out
}
// Encodes parity shards for a set of data shards.
//
// Input is 'shards' containing readers for data shards followed by parity shards
// io.Writer.
//
// The number of shards must match the number given to NewStream().
//
// Each reader must supply the same number of bytes.
//
// The parity shards will be written to the writer.
// The number of bytes written will match the input size.
//
// If a data stream returns an error, a StreamReadError type error
// will be returned. If a parity writer returns an error, a
// StreamWriteError will be returned.
func (r rsStream) Encode(data []io.Reader, parity []io.Writer) error {
if len(data) != r.r.DataShards {
return ErrTooFewShards
}
if len(parity) != r.r.ParityShards {
return ErrTooFewShards
}
all := createSlice(r.r.Shards, r.bs)
in := all[:r.r.DataShards]
out := all[r.r.DataShards:]
read := 0
for {
err := r.readShards(in, data)
switch err {
case nil:
case io.EOF:
if read == 0 {
return ErrShardNoData
}
return nil
default:
return err
}
out = trimShards(out, shardSize(in))
read += shardSize(in)
err = r.r.Encode(all)
if err != nil {
return err
}
err = r.writeShards(parity, out)
if err != nil {
return err
}
}
}
// Trim the shards so they are all the same size
func trimShards(in [][]byte, size int) [][]byte {
for i := range in {
if in[i] != nil {
in[i] = in[i][0:size]
}
if len(in[i]) < size {
in[i] = nil
}
}
return in
}
func readShards(dst [][]byte, in []io.Reader) error {
if len(in) != len(dst) {
panic("internal error: in and dst size does not match")
}
size := -1
for i := range in {
if in[i] == nil {
dst[i] = nil
continue
}
n, err := io.ReadFull(in[i], dst[i])
// The error is EOF only if no bytes were read.
// If an EOF happens after reading some but not all the bytes,
// ReadFull returns ErrUnexpectedEOF.
switch err {
case io.ErrUnexpectedEOF, io.EOF:
if size < 0 {
size = n
} else if n != size {
// Shard sizes must match.
return ErrShardSize
}
dst[i] = dst[i][0:n]
case nil:
continue
default:
return StreamReadError{Err: err, Stream: i}
}
}
if size == 0 {
return io.EOF
}
return nil
}
func writeShards(out []io.Writer, in [][]byte) error {
if len(out) != len(in) {
panic("internal error: in and out size does not match")
}
for i := range in {
if out[i] == nil {
continue
}
n, err := out[i].Write(in[i])
if err != nil {
return StreamWriteError{Err: err, Stream: i}
}
//
if n != len(in[i]) {
return StreamWriteError{Err: io.ErrShortWrite, Stream: i}
}
}
return nil
}
type readResult struct {
n int
size int
err error
}
// cReadShards reads shards concurrently
func cReadShards(dst [][]byte, in []io.Reader) error {
if len(in) != len(dst) {
panic("internal error: in and dst size does not match")
}
var wg sync.WaitGroup
wg.Add(len(in))
res := make(chan readResult, len(in))
for i := range in {
if in[i] == nil {
dst[i] = nil
wg.Done()
continue
}
go func(i int) {
defer wg.Done()
n, err := io.ReadFull(in[i], dst[i])
// The error is EOF only if no bytes were read.
// If an EOF happens after reading some but not all the bytes,
// ReadFull returns ErrUnexpectedEOF.
res <- readResult{size: n, err: err, n: i}
}(i)
}
wg.Wait()
close(res)
size := -1
for r := range res {
switch r.err {
case io.ErrUnexpectedEOF, io.EOF:
if size < 0 {
size = r.size
} else if r.size != size {
// Shard sizes must match.
return ErrShardSize
}
dst[r.n] = dst[r.n][0:r.size]
case nil:
default:
return StreamReadError{Err: r.err, Stream: r.n}
}
}
if size == 0 {
return io.EOF
}
return nil
}
// cWriteShards writes shards concurrently
func cWriteShards(out []io.Writer, in [][]byte) error {
if len(out) != len(in) {
panic("internal error: in and out size does not match")
}
var errs = make(chan error, len(out))
var wg sync.WaitGroup
wg.Add(len(out))
for i := range in {
go func(i int) {
defer wg.Done()
if out[i] == nil {
errs <- nil
return
}
n, err := out[i].Write(in[i])
if err != nil {
errs <- StreamWriteError{Err: err, Stream: i}
return
}
if n != len(in[i]) {
errs <- StreamWriteError{Err: io.ErrShortWrite, Stream: i}
}
}(i)
}
wg.Wait()
close(errs)
for err := range errs {
if err != nil {
return err
}
}
return nil
}
// Verify returns true if the parity shards contain correct data.
//
// The number of shards must match the number total data+parity shards
// given to NewStream().
//
// Each reader must supply the same number of bytes.
// If a shard stream returns an error, a StreamReadError type error
// will be returned.
func (r rsStream) Verify(shards []io.Reader) (bool, error) {
if len(shards) != r.r.Shards {
return false, ErrTooFewShards
}
read := 0
all := createSlice(r.r.Shards, r.bs)
for {
err := r.readShards(all, shards)
if err == io.EOF {
if read == 0 {
return false, ErrShardNoData
}
return true, nil
}
if err != nil {
return false, err
}
read += shardSize(all)
ok, err := r.r.Verify(all)
if !ok || err != nil {
return ok, err
}
}
}
// ErrReconstructMismatch is returned by the StreamEncoder, if you supply
// "valid" and "fill" streams on the same index.
// Therefore it is impossible to see if you consider the shard valid
// or would like to have it reconstructed.
var ErrReconstructMismatch = errors.New("valid shards and fill shards are mutually exclusive")
// Reconstruct will recreate the missing shards if possible.
//
// Given a list of valid shards (to read) and invalid shards (to write)
//
// You indicate that a shard is missing by setting it to nil in the 'valid'
// slice and at the same time setting a non-nil writer in "fill".
// An index cannot contain both non-nil 'valid' and 'fill' entry.
//
// If there are too few shards to reconstruct the missing
// ones, ErrTooFewShards will be returned.
//
// The reconstructed shard set is complete, but integrity is not verified.
// Use the Verify function to check if data set is ok.
func (r rsStream) Reconstruct(valid []io.Reader, fill []io.Writer) error {
if len(valid) != r.r.Shards {
return ErrTooFewShards
}
if len(fill) != r.r.Shards {
return ErrTooFewShards
}
all := createSlice(r.r.Shards, r.bs)
for i := range valid {
if valid[i] != nil && fill[i] != nil {
return ErrReconstructMismatch
}
}
read := 0
for {
err := r.readShards(all, valid)
if err == io.EOF {
if read == 0 {
return ErrShardNoData
}
return nil
}
if err != nil {
return err
}
read += shardSize(all)
all = trimShards(all, shardSize(all))
err = r.r.Reconstruct(all)
if err != nil {
return err
}
err = r.writeShards(fill, all)
if err != nil {
return err
}
}
}
// Join the shards and write the data segment to dst.
//
// Only the data shards are considered.
//
// You must supply the exact output size you want.
// If there are to few shards given, ErrTooFewShards will be returned.
// If the total data size is less than outSize, ErrShortData will be returned.
func (r rsStream) Join(dst io.Writer, shards []io.Reader, outSize int64) error {
// Do we have enough shards?
if len(shards) < r.r.DataShards {
return ErrTooFewShards
}
// Trim off parity shards if any
shards = shards[:r.r.DataShards]
for i := range shards {
if shards[i] == nil {
return StreamReadError{Err: ErrShardNoData, Stream: i}
}
}
// Join all shards
src := io.MultiReader(shards...)
// Copy data to dst
n, err := io.CopyN(dst, src, outSize)
if err == io.EOF {
return ErrShortData
}
if err != nil {
return err
}
if n != outSize {
return ErrShortData
}
return nil
}
// Split a an input stream into the number of shards given to the encoder.
//
// The data will be split into equally sized shards.
// If the data size isn't dividable by the number of shards,
// the last shard will contain extra zeros.
//
// You must supply the total size of your input.
// 'ErrShortData' will be returned if it is unable to retrieve the
// number of bytes indicated.
func (r rsStream) Split(data io.Reader, dst []io.Writer, size int64) error {
if size == 0 {
return ErrShortData
}
if len(dst) != r.r.DataShards {
return ErrInvShardNum
}
for i := range dst {
if dst[i] == nil {
return StreamWriteError{Err: ErrShardNoData, Stream: i}
}
}
// Calculate number of bytes per shard.
perShard := (size + int64(r.r.DataShards) - 1) / int64(r.r.DataShards)
// Pad data to r.Shards*perShard.
padding := make([]byte, (int64(r.r.Shards)*perShard)-size)
data = io.MultiReader(data, bytes.NewBuffer(padding))
// Split into equal-length shards and copy.
for i := range dst {
n, err := io.CopyN(dst[i], data, perShard)
if err != io.EOF && err != nil {
return err
}
if n != perShard {
return ErrShortData
}
}
return nil
}

View File

@@ -181,7 +181,7 @@ func indentMessageLines(message string, longestLabelLen int) string {
// no need to align first line because it starts at the correct location (after the label)
if i != 0 {
// append alignLen+1 spaces to align with "{{longestLabel}}:" before adding tab
outBuf.WriteString("\n\r\t" + strings.Repeat(" ", longestLabelLen+1) + "\t")
outBuf.WriteString("\n\r\t" + strings.Repeat(" ", longestLabelLen +1) + "\t")
}
outBuf.WriteString(scanner.Text())
}
@@ -229,7 +229,7 @@ func Fail(t TestingT, failureMessage string, msgAndArgs ...interface{}) bool {
}
type labeledContent struct {
label string
label string
content string
}

24
vendor/github.com/xtaci/kcp-go/.gitignore generated vendored Normal file
View File

@@ -0,0 +1,24 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof

15
vendor/github.com/xtaci/kcp-go/.travis.yml generated vendored Normal file
View File

@@ -0,0 +1,15 @@
language: go
go:
- 1.8
before_install:
- go get -t -v ./...
install:
- go get github.com/xtaci/kcp-go
script:
- go test -coverprofile=coverage.txt -covermode=atomic -bench .
after_success:
- bash <(curl -s https://codecov.io/bash)

Some files were not shown because too many files have changed in this diff Show More