Ceph对象存储RGW简单安装

作者: sysit 分类: d 发表于 2019-03-01 417人围观

Ceph RGW基于librados，是为应用提供RESTful类型的对象存储接口。RGW提供两种类型的接口： 
　　1) S3：兼容Amazon S3RESTful API； 
　　2) Swift：兼容OpenStack Swift API。 
![title](/api/file/getImage?fileId=5c77981b3cd7c91e15000560)
　　S3和Swift API共享同一个命名空间，所以可以使用两种API访问相同的数据。
　　
# 1. 前置安装
前置要求安装完成CEPH集群，参考本站《Ceph安装配置手册》

# 2. 安装RGW
```
# 安装rgw包
ceph-deploy install --rgw node1 node2 node3 --no-adjust-repos
# --no-adjust-repos是直接使用本地源，不生成官方源。

# 推送key
ceph-deploy admin node1 node2 node3

# 初始化
ceph-deploy rgw create node1 node2 node3

# 初始化完成之后，当前路径下多了一个keyring文件
#ceph.bootstrap-rgw.keyring

#查看
netstat -antlp |grep radosgw
#radosgw可以看到默认监听到7480端口上

#修改radosgw默认配置

[root@node1 ~]# su - admin
[admin@node1 ~]$ cd my-cluster/

vi ceph.conf

#添加
[client.rgw.node1]
rgw_frontends = "civetweb port=80 num_threads=20"
rgw dns name = s3.sysit.cn
rgw_thread_pool_size = 1000
rgw_curl_wait_timeout_ms = 100000
rgw_override_bucket_index_max_shards = 3
rgw_max_chunk_size = 1048576
rgw_cache_lru_size = 1000000
rgw_bucket_default_quota_max_objects = 10000000
[client.rgw.node2]
rgw_frontends = "civetweb port=80 num_threads=20"
rgw dns name = s3.sysit.cn
rgw_thread_pool_size = 1000
rgw_curl_wait_timeout_ms = 100000
rgw_override_bucket_index_max_shards = 3
rgw_max_chunk_size = 1048576
rgw_cache_lru_size = 1000000
rgw_bucket_default_quota_max_objects = 10000000
[client.rgw.node3]
rgw_frontends = "civetweb port=80 num_threads=20"
rgw dns name = s3.sysit.cn
rgw_thread_pool_size = 1000
rgw_curl_wait_timeout_ms = 100000
rgw_override_bucket_index_max_shards = 3
rgw_max_chunk_size = 1048576
rgw_cache_lru_size = 1000000
rgw_bucket_default_quota_max_objects = 10000000

# 推送配置
ceph-deploy --overwrite-conf config push node1 node2 node3

# 重启服务
systemctl restart ceph-radosgw.target
```
# 3. 验证
将s3.sysit.cn分别解析到以上节点，在浏览器中访问，出现如下信息：

![title](/api/file/getImage?fileId=5c789eed3cd7c91e15000568)

# 4. 配置s3的nginx负载均衡
```
vi /etc/nginx/conf.d/s3.conf

upstream s3
        {
            ip_hash;
            server node1:80;
            server node2:80;
            server node3:80;
        }

server {
        listen       80;
        server_name  s3.bbdops.com;

location / {
           proxy_pass http://s3;
           proxy_redirect off;
           proxy_set_header HOST $host;
           proxy_set_header X-Real-IP $remote_addr;
           proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
           client_max_body_size 5120m;
           proxy_buffering off;
           }

error_page   500 502 503 504  /50x.html;
           location = /50x.html {
           root   html;
           }
}
server {
        listen 443 ssl;
        listen [::]:443 ssl;
        server_name s3.bbdops.com *.s3.bbdops.com;

ssl_certificate /usr/local/nginx/conf/server.crt;
        ssl_certificate_key /usr/local/nginx/conf/server_nopwd.key;
        add_header Strict-Transport-Security 'max-age=31536000; preload';

location / {
                proxy_redirect off;
                proxy_pass http://s3;
                proxy_buffering off;
                 proxy_set_header Host $http_host;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                proxy_set_header X-Forwarded-Proto $scheme;
        }
}

#配置bucket的泛域名负载均衡支持
vi /etc/nginx/conf.d/bucket-s3.conf
upstream bucket-s3
        {
            ip_hash;
            server node1:80;
            server node2:80;
            server node3:80;
        }

server {
        listen       80;
        server_name  *.s3.sysit.cn;

location / {
           proxy_pass http://bucket-s3;
           proxy_redirect off;
           proxy_set_header HOST $host;
           proxy_set_header X-Real-IP $remote_addr;
           proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
           client_max_body_size 5120m;
           proxy_buffering off;
           }

error_page   500 502 503 504  /50x.html;
           location = /50x.html {
           root   html;
           }
}
```
# 5. 用户

```
[root@node1 ~]# radosgw-admin user create --uid=test --display-name="test" 
{
    "user_id": "test",
    "display_name": "test",
    "email": "",
    "suspended": 0,
    "max_buckets": 1000,
    "auid": 0,
    "subusers": [],
    "keys": [
        {
            "user": "test",
            "access_key": "2HHV0BWNYH838EZ644DC",
            "secret_key": "VuqASI2SKMx2u021KSVtZoGgrk3kF1vOYIg8J2cw"
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "default_placement": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "temp_url_keys": [],
    "type": "rgw",
    "mfa_ids": []
}
```

# 6. 在dashboard上启用rgw

```
# 在dashbaord上管理用户和bucket，需要在radosgw中创建一个用户凭据，并且这个用户需要有system的权限

radosgw-admin user create --uid=admin --display-name=admin --system
{
    "user_id": "admin",
    "display_name": "admin",
    "email": "",
    "suspended": 0,
    "max_buckets": 1000,
    "auid": 0,
    "subusers": [],
    "keys": [
        {
            "user": "admin",
            "access_key": "T9RD5BRM7ZRCUROZ7Z26",
            "secret_key": "qvCUpTZOwdO20QIvj3hQu1sPKpO64Go1CnNWEKs0"
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "system": "true",
    "default_placement": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "temp_url_keys": [],
    "type": "rgw",
    "mfa_ids": []
}

# 可以查看用户
radosgw-admin user info --uid=admin
{
    "user_id": "admin",
    "display_name": "admin",
    "email": "",
    "suspended": 0,
    "max_buckets": 1000,
    "auid": 0,
    "subusers": [],
    "keys": [
        {
            "user": "admin",
            "access_key": "T9RD5BRM7ZRCUROZ7Z26",
            "secret_key": "qvCUpTZOwdO20QIvj3hQu1sPKpO64Go1CnNWEKs0"
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "system": "true",
    "default_placement": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "temp_url_keys": [],
    "type": "rgw",
    "mfa_ids": []
}

#将admin用户的access_key 和secret_key提供给dashboard
#ceph dashboard set-rgw-api-access-key <access_key>
#ceph dashboard set-rgw-api-secret-key <secret_key>
[root@node1 ~]# ceph dashboard set-rgw-api-access-key T9RD5BRM7ZRCUROZ7Z26
Option RGW_API_ACCESS_KEY updated

[root@node1 ~]# ceph dashboard set-rgw-api-secret-key qvCUpTZOwdO20QIvj3hQu1sPKpO64Go1CnNWEKs0
Option RGW_API_SECRET_KEY updated

```

以上就可以在dashboard上操作bucket和user了

user:
![title](/api/file/getImage?fileId=5c77a95a3cd7c91e15000561)

![title](/api/file/getImage?fileId=5c77a99c3cd7c91e15000562)

![title](/api/file/getImage?fileId=5c78a0253cd7c91e1500056b)

bucket:

![title](/api/file/getImage?fileId=5c789fa83cd7c91e15000569)

![title](/api/file/getImage?fileId=5c789fc73cd7c91e1500056a)

如果觉得我的文章对您有用，请随意赞赏。您的支持将鼓励我继续创作！