通过 Nginx 代理转发TOS SDK 请求

本文介绍 SDK 通过 Nginx 代理访问 TOS 的过程。

当客户端位于火山网络外，且大部分客户端无法访问外网，如客户端位于客户本地机房且无外网访问权限时，可以通过搭建代理服务器，将 SDK 的请求通过代理发送到 TOS 服务端。

TOS SDK 支持设置 HTTP 协议代理服务发送请求（目前只支持 HTTP 协议代理）。

本实验使用Go SDK跟 Nginx 代理服务实现代理转发SDK请求。

• 预计部署时间：30分钟
• 级别：初级
• 相关产品：TOS
• 受众： 通用

• 点击此链接登录控制台。

• 如果您还没有账户，请点击此链接注册账户。

一、安装Nginx

您可以使用如下命令安装Nginx并启动服务。

yum install nginx -y
systemctl start nginx
systemctl status nginx

安装成功后，进程状态如下：

[root@iv-xxxx ~]# systemctl status nginx
● nginx.service - The nginx HTTP and reverse proxy server
   Loaded: loaded (/usr/lib/systemd/system/nginx.service; disabled; vendor preset: disabled)
   Active: active (running) since 四 2024-01-04 14:06:00 CST; 24min ago
  Process: 1305 ExecStart=/usr/sbin/nginx (code=exited, status=0/SUCCESS)
  Process: 1302 ExecStartPre=/usr/sbin/nginx -t (code=exited, status=0/SUCCESS)
  Process: 1300 ExecStartPre=/usr/bin/rm -f /run/nginx.pid (code=exited, status=0/SUCCESS)
 Main PID: 1307 (nginx)
    Tasks: 3
   Memory: 2.1M
   CGroup: /system.slice/nginx.service
           ├─1307 nginx: master process /usr/sbin/nginx
           ├─1308 nginx: worker process
           └─1309 nginx: worker process

1月 04 14:06:00 iv-xxxx systemd[1]: Starting The nginx HTTP and reverse proxy server...
1月 04 14:06:00 iv-xxxx nginx[1302]: nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
1月 04 14:06:00 iv-xxxx nginx[1302]: nginx: configuration file /etc/nginx/nginx.conf test is successful
1月 04 14:06:00 iv-xxxx systemd[1]: Started The nginx HTTP and reverse proxy server.

配置文件如下：

[root@iv-xxxx nginx]# cat nginx.conf
# For more information on configuration, see:
#   * Official English Documentation: http://nginx.org/en/docs/
#   * Official Russian Documentation: http://nginx.org/ru/docs/

user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;

# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic.
include /usr/share/nginx/modules/*.conf;

events {
    worker_connections 1024;
}

http {
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;

    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   65;
    types_hash_max_size 4096;

    include             /etc/nginx/mime.types;
    default_type        application/octet-stream;

    # Load modular configuration files from the /etc/nginx/conf.d directory.
    # See http://nginx.org/en/docs/ngx_core_module.html#include
    # for more information.
    include /etc/nginx/conf.d/*.conf;
    client_max_body_size 100M;
    server {
        listen       80;
        listen       [::]:80;
        server_name  xxx.xxx.xxx.xxx;
        # root         /usr/share/nginx/html;

        # Load configuration files for the default server block.
        include /etc/nginx/default.d/*.conf;

        location = / {
                proxy_pass http://tos-cn-beijing.volces.com;
        }
        location / {
                proxy_pass http://tos-cn-beijing.volces.com;
                proxy_set_header Host $host;
        }

        }
}

本配置文件只做示例演示，您可以根据实际环境调整配置文件。

二、代码实现

本示例代码使用TOS Go SDK完成，示例为上传对象至TOS。

示例代码如下：

package main

import (
	"context"
	"fmt"
	"github.com/volcengine/ve-tos-golang-sdk/v2/tos"
	"github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum"
	"io"
	"os"
)

func checkErr1(err error) {
	if err != nil {
		if serverErr, ok := err.(*tos.TosServerError); ok {
			fmt.Println("Error:", serverErr.Error())
			fmt.Println("Request ID:", serverErr.RequestID)
			fmt.Println("Response Status Code:", serverErr.StatusCode)
			fmt.Println("Response Header:", serverErr.Header)
			fmt.Println("Response Err Code:", serverErr.Code)
			fmt.Println("Response Err Msg:", serverErr.Message)
		} else if clientErr, ok := err.(*tos.TosClientError); ok {
			fmt.Println("Error:", clientErr.Error())
			fmt.Println("Client Cause Err:", clientErr.Cause.Error())
		} else {
			fmt.Println("Error:", err)
		}
		//panic(err)
	}
}

func main() {
	var (
		ak       = "xxxx"
		sk       = "xxxx"
		endpoint = "http://tos-cn-beijing.volces.com"
		region   = "cn-beijing"

		proxyHost = "http://xxx.xxx.xxx.xxx"
		proxyPort = 80 /* Provide your proxy port */
	)

	proxy, err := tos.NewProxy(proxyHost, proxyPort)
	checkErr1(err)
	credential := tos.NewStaticCredentials(ak, sk)
	client, err := tos.NewClientV2(endpoint, tos.WithCredentials(credential), tos.WithRegion(region), tos.WithProxy(proxy))
	checkErr1(err)

	// listbuckets
	resp, err1 := client.ListBuckets(context.Background(), &tos.ListBucketsInput{})
	if err1 != nil {
		fmt.Println(err1)
	}
	fmt.Println(resp)

	// getobject
	//inputstruct := &tos.GetObjectV2Input{
	//	Bucket: "xxxx",
	//	Key:    "xxxx",
	//}
	//
	//getresp, err1 := client.GetObjectV2(context.Background(), inputstruct)
	//
	//if err1 != nil {
	//	fmt.Println(err1)
	//}
	//
	//fmt.Println(getresp)

	// putobject
	//file, _ := os.Open("/Users/bytedance/Downloads/log1.txt")
	//
	//inputstruct := tos.PutObjectBasicInput{
	//	Bucket: "xxxx",
	//	Key:    "xxxx",
	//}
	//
	//putresp, err1 := client.PutObjectV2(context.Background(), &tos.PutObjectV2Input{
	//	inputstruct,
	//	file,
	//})
	//
	//checkErr1(err1)
	//client.Close()
	// 上述注释的getobject跟putobject 可以分开进行测试

	// multipartupload 分片上传

	bucketName := "xxxx"

	// 指定的 ObjectKey
	objectKey := "xxxx"

	createMultipartOutput, err := client.CreateMultipartUploadV2(context.Background(), &tos.CreateMultipartUploadV2Input{
		Bucket:       bucketName,
		Key:          objectKey,
		ACL:          enum.ACLPrivate,
		StorageClass: enum.StorageClassIa,
		Meta:         map[string]string{"key": "value"},
	})

	checkErr1(err)
	fmt.Println("CreateMultipartUploadV2 Request ID:", createMultipartOutput.RequestID)
	// 获取到上传的 UploadID
	fmt.Println("CreateMultipartUploadV2 Upload ID:", createMultipartOutput.UploadID)
	// 需要上传的文件路径
	localFile := "/Users/bytedance/Documents/配套资料文档.zip"
	fd, err := os.Open(localFile)
	checkErr1(err)
	defer fd.Close()
	stat, err := os.Stat(localFile)
	checkErr1(err)
	fileSize := stat.Size()
	// partNumber 编号从 1 开始
	partNumber := 1
	// part size 大小设置为 20M
	partSize := int64(20 * 1024 * 1024)
	offset := int64(0)
	var parts []tos.UploadedPartV2
	for offset < fileSize {
		uploadSize := partSize
		// 最后一个分片
		if fileSize-offset < partSize {
			uploadSize = fileSize - offset
		}
		fd.Seek(offset, io.SeekStart)
		partOutput, errupload := client.UploadPartV2(context.Background(), &tos.UploadPartV2Input{
			UploadPartBasicInput: tos.UploadPartBasicInput{
				Bucket:     bucketName,
				Key:        objectKey,
				UploadID:   createMultipartOutput.UploadID,
				PartNumber: partNumber,
			},
			Content:       io.LimitReader(fd, uploadSize),
			ContentLength: uploadSize,
		})

		checkErr1(errupload)

		fmt.Println("upload Request ID:", partOutput.RequestID)
		parts = append(parts, tos.UploadedPartV2{PartNumber: partNumber, ETag: partOutput.ETag})
		offset += uploadSize
		partNumber++
	}

	completeOutput, err := client.CompleteMultipartUploadV2(context.Background(), &tos.CompleteMultipartUploadV2Input{
		Bucket:   bucketName,
		Key:      objectKey,
		UploadID: createMultipartOutput.UploadID,
		Parts:    parts,
	})
	checkErr1(err)
	fmt.Println("CompleteMultipartUploadV2 Request ID:", completeOutput.RequestID)
}

Nginx 日志如下：

114.251.196.85 - - [04/Jan/2024:14:34:13 +0800] "POST http://.tos-cn-beijing.volces.com/xxxx?uploads= HTTP/1.1" 200 80 "-" "tos-go-sdk/v2.6.3 (darwin/amd64;go1.19.4)" "-"
114.251.196.85 - - [04/Jan/2024:14:34:20 +0800] "PUT http://xxxx.tos-cn-beijing.volces.com/xxxx?partNumber=1&uploadId=7443ad5c0d03e6306ea0000001655ace HTTP/1.1" 200 0 "-" "tos-go-sdk/v2.6.3 (darwin/amd64;go1.19.4)" "-"
114.251.196.85 - - [04/Jan/2024:14:34:27 +0800] "PUT http://xxxx.tos-cn-beijing.volces.com/xxxx?partNumber=2&uploadId=7443ad5c0d03e6306ea0000001655ace HTTP/1.1" 200 0 "-" "tos-go-sdk/v2.6.3 (darwin/amd64;go1.19.4)" "-"
114.251.196.85 - - [04/Jan/2024:14:34:33 +0800] "PUT http://xxxx.tos-cn-beijing.volces.com/xxxx?partNumber=3&uploadId=7443ad5c0d03e6306ea0000001655ace HTTP/1.1" 200 0 "-" "tos-go-sdk/v2.6.3 (darwin/amd64;go1.19.4)" "-"
114.251.196.85 - - [04/Jan/2024:14:34:40 +0800] "PUT http://xxxx.tos-cn-beijing.volces.com/xxxx?partNumber=4&uploadId=7443ad5c0d03e6306ea0000001655ace HTTP/1.1" 200 0 "-" "tos-go-sdk/v2.6.3 (darwin/amd64;go1.19.4)" "-"
114.251.196.85 - - [04/Jan/2024:14:34:45 +0800] "PUT http://xxxx.tos-cn-beijing.volces.com/xxxx?partNumber=5&uploadId=7443ad5c0d03e6306ea0000001655ace HTTP/1.1" 200 0 "-" "tos-go-sdk/v2.6.3 (darwin/amd64;go1.19.4)" "-"
114.251.196.85 - - [04/Jan/2024:14:34:45 +0800] "POST http://xxxx.tos-cn-beijing.volces.com/xxxx?uploadId=7443ad5c0d03e6306ea0000001655ace HTTP/1.1" 200 142 "-" "tos-go-sdk/v2.6.3 (darwin/amd64;go1.19.4)" "-"

可以看到 nginx 日志响应正常，然后可以到控制台查看此对象已经上传成功，如下；

参考文档：
[1] TOS Go SDK