Ceph对象存储RGW简单安装 作者: sysit 分类: d 发表于 2019-03-01 417人围观 Ceph RGW基于librados,是为应用提供RESTful类型的对象存储接口。RGW提供两种类型的接口: 1) S3:兼容Amazon S3RESTful API; 2) Swift:兼容OpenStack Swift API。  S3和Swift API共享同一个命名空间,所以可以使用两种API访问相同的数据。 # 1. 前置安装 前置要求安装完成CEPH集群,参考本站《Ceph安装配置手册》 # 2. 安装RGW ``` # 安装rgw包 ceph-deploy install --rgw node1 node2 node3 --no-adjust-repos # --no-adjust-repos是直接使用本地源,不生成官方源。 # 推送key ceph-deploy admin node1 node2 node3 # 初始化 ceph-deploy rgw create node1 node2 node3 # 初始化完成之后,当前路径下多了一个keyring文件 #ceph.bootstrap-rgw.keyring #查看 netstat -antlp |grep radosgw #radosgw可以看到默认监听到7480端口上 #修改radosgw默认配置 [root@node1 ~]# su - admin [admin@node1 ~]$ cd my-cluster/ vi ceph.conf #添加 [client.rgw.node1] rgw_frontends = "civetweb port=80 num_threads=20" rgw dns name = s3.sysit.cn rgw_thread_pool_size = 1000 rgw_curl_wait_timeout_ms = 100000 rgw_override_bucket_index_max_shards = 3 rgw_max_chunk_size = 1048576 rgw_cache_lru_size = 1000000 rgw_bucket_default_quota_max_objects = 10000000 [client.rgw.node2] rgw_frontends = "civetweb port=80 num_threads=20" rgw dns name = s3.sysit.cn rgw_thread_pool_size = 1000 rgw_curl_wait_timeout_ms = 100000 rgw_override_bucket_index_max_shards = 3 rgw_max_chunk_size = 1048576 rgw_cache_lru_size = 1000000 rgw_bucket_default_quota_max_objects = 10000000 [client.rgw.node3] rgw_frontends = "civetweb port=80 num_threads=20" rgw dns name = s3.sysit.cn rgw_thread_pool_size = 1000 rgw_curl_wait_timeout_ms = 100000 rgw_override_bucket_index_max_shards = 3 rgw_max_chunk_size = 1048576 rgw_cache_lru_size = 1000000 rgw_bucket_default_quota_max_objects = 10000000 # 推送配置 ceph-deploy --overwrite-conf config push node1 node2 node3 # 重启服务 systemctl restart ceph-radosgw.target ``` # 3. 验证 将s3.sysit.cn分别解析到以上节点,在浏览器中访问,出现如下信息:  # 4. 配置s3的nginx负载均衡 ``` vi /etc/nginx/conf.d/s3.conf upstream s3 { ip_hash; server node1:80; server node2:80; server node3:80; } server { listen 80; server_name s3.bbdops.com; location / { proxy_pass http://s3; proxy_redirect off; proxy_set_header HOST $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; client_max_body_size 5120m; proxy_buffering off; } error_page 500 502 503 504 /50x.html; location = /50x.html { root html; } } server { listen 443 ssl; listen [::]:443 ssl; server_name s3.bbdops.com *.s3.bbdops.com; ssl_certificate /usr/local/nginx/conf/server.crt; ssl_certificate_key /usr/local/nginx/conf/server_nopwd.key; add_header Strict-Transport-Security 'max-age=31536000; preload'; location / { proxy_redirect off; proxy_pass http://s3; proxy_buffering off; proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; } } #配置bucket的泛域名负载均衡支持 vi /etc/nginx/conf.d/bucket-s3.conf upstream bucket-s3 { ip_hash; server node1:80; server node2:80; server node3:80; } server { listen 80; server_name *.s3.sysit.cn; location / { proxy_pass http://bucket-s3; proxy_redirect off; proxy_set_header HOST $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; client_max_body_size 5120m; proxy_buffering off; } error_page 500 502 503 504 /50x.html; location = /50x.html { root html; } } ``` # 5. 用户 ``` [root@node1 ~]# radosgw-admin user create --uid=test --display-name="test" { "user_id": "test", "display_name": "test", "email": "", "suspended": 0, "max_buckets": 1000, "auid": 0, "subusers": [], "keys": [ { "user": "test", "access_key": "2HHV0BWNYH838EZ644DC", "secret_key": "VuqASI2SKMx2u021KSVtZoGgrk3kF1vOYIg8J2cw" } ], "swift_keys": [], "caps": [], "op_mask": "read, write, delete", "default_placement": "", "placement_tags": [], "bucket_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "user_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "temp_url_keys": [], "type": "rgw", "mfa_ids": [] } ``` # 6. 在dashboard上启用rgw ``` # 在dashbaord上管理用户和bucket,需要在radosgw中创建一个用户凭据,并且这个用户需要有system的权限 radosgw-admin user create --uid=admin --display-name=admin --system { "user_id": "admin", "display_name": "admin", "email": "", "suspended": 0, "max_buckets": 1000, "auid": 0, "subusers": [], "keys": [ { "user": "admin", "access_key": "T9RD5BRM7ZRCUROZ7Z26", "secret_key": "qvCUpTZOwdO20QIvj3hQu1sPKpO64Go1CnNWEKs0" } ], "swift_keys": [], "caps": [], "op_mask": "read, write, delete", "system": "true", "default_placement": "", "placement_tags": [], "bucket_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "user_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "temp_url_keys": [], "type": "rgw", "mfa_ids": [] } # 可以查看用户 radosgw-admin user info --uid=admin { "user_id": "admin", "display_name": "admin", "email": "", "suspended": 0, "max_buckets": 1000, "auid": 0, "subusers": [], "keys": [ { "user": "admin", "access_key": "T9RD5BRM7ZRCUROZ7Z26", "secret_key": "qvCUpTZOwdO20QIvj3hQu1sPKpO64Go1CnNWEKs0" } ], "swift_keys": [], "caps": [], "op_mask": "read, write, delete", "system": "true", "default_placement": "", "placement_tags": [], "bucket_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "user_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "temp_url_keys": [], "type": "rgw", "mfa_ids": [] } #将admin用户的access_key 和secret_key提供给dashboard #ceph dashboard set-rgw-api-access-key <access_key> #ceph dashboard set-rgw-api-secret-key <secret_key> [root@node1 ~]# ceph dashboard set-rgw-api-access-key T9RD5BRM7ZRCUROZ7Z26 Option RGW_API_ACCESS_KEY updated [root@node1 ~]# ceph dashboard set-rgw-api-secret-key qvCUpTZOwdO20QIvj3hQu1sPKpO64Go1CnNWEKs0 Option RGW_API_SECRET_KEY updated ``` 以上就可以在dashboard上操作bucket和user了 user:    bucket:   如果觉得我的文章对您有用,请随意赞赏。您的支持将鼓励我继续创作! 赞赏支持