Merge pull request #431 from qiniu/fix/list-domains

update list_domains and get_bucket_hosts

Author: bachue

.github/workflows/ci-test.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install flake8 pytest pytest-cov freezegun requests scrutinizer-ocular codecov
+          pip install "coverage<7.2" flake8 pytest pytest-cov freezegun requests scrutinizer-ocular codecov
       - name: Run cases
         env:
           QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }}

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## Next Version
+* 对象存储，更新 api 默认域名
+* 对象存储，新增 api 域名的配置与获取
+* 对象存储，修复获取区域域名后无法按照预期进行过期处理
+* 对象存储，更新获取区域域名的接口
+* 对象存储，bucket_domains 修改为 list_domains 的别名
+
 ## 7.10.0（2022-11-15）
 * 对象存储，修复通过 set_default 设置 rs, rsf 不生效，而 SDK 自动获取的问题（v7.9.0）
 * 对象存储，支持直接从 qiniu 导入 UploadProgressRecorder

qiniu/config.py

@@ -4,7 +4,7 @@
 
 RS_HOST = 'http://rs.qiniu.com'  # 管理操作Host
 RSF_HOST = 'http://rsf.qbox.me'  # 列举操作Host
-API_HOST = 'http://api.qiniu.com'  # 数据处理操作Host
+API_HOST = 'http://api.qiniuapi.com'  # 数据处理操作Host
 UC_HOST = 'https://uc.qbox.me'  # 获取空间信息Host
 
 _BLOCK_SIZE = 1024 * 1024 * 4  # 断点续传分块大小，该参数为接口规格，暂不支持修改

qiniu/region.py

@@ -23,13 +23,15 @@ def __init__(
             home_dir=None,
             scheme="http",
             rs_host=None,
-            rsf_host=None):
+            rsf_host=None,
+            api_host=None):
         """初始化Zone类"""
         self.up_host = up_host
         self.up_host_backup = up_host_backup
         self.io_host = io_host
         self.rs_host = rs_host
         self.rsf_host = rsf_host
+        self.api_host = api_host
         self.home_dir = home_dir
         self.host_cache = host_cache
         self.scheme = scheme
@@ -91,6 +93,20 @@ def get_rsf_host(self, ak, bucket, home_dir=None):
         rsf_hosts = bucket_hosts['rsfHosts']
         return rsf_hosts[0]
 
+    def get_api_host(self, ak, bucket, home_dir=None):
+        from .config import get_default, is_customized_default
+        if self.api_host:
+            return self.api_host
+        if is_customized_default('default_api_host'):
+            return get_default('default_api_host')
+        if home_dir is None:
+            home_dir = os.getcwd()
+        bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir)
+        if 'apiHosts' not in bucket_hosts:
+            bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir, force=True)
+        api_hosts = bucket_hosts['apiHosts']
+        return api_hosts[0]
+
     def get_up_host(self, ak, bucket, home_dir):
         bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir)
         if 'upHosts' not in bucket_hosts:
@@ -123,39 +139,38 @@ def get_bucket_hosts(self, ak, bucket, home_dir, force=False):
         if not force and len(bucket_hosts) > 0:
             return bucket_hosts
 
-        hosts = {}
-        hosts.update({self.scheme: {}})
-
-        hosts[self.scheme].update({'up': []})
-        hosts[self.scheme].update({'io': []})
+        hosts = compat.json.loads(self.bucket_hosts(ak, bucket)).get('hosts', [])
 
-        if self.up_host is not None:
-            hosts[self.scheme]['up'].append(self.scheme + "://" + self.up_host)
+        if type(hosts) is not list or len(hosts) == 0:
+            raise KeyError("Please check your BUCKET_NAME! Server hosts not correct! The hosts is %s" % hosts)
 
-        if self.up_host_backup is not None:
-            hosts[self.scheme]['up'].append(
-                self.scheme + "://" + self.up_host_backup)
+        region = hosts[0]
 
-        if self.io_host is not None:
-            hosts[self.scheme]['io'].append(self.scheme + "://" + self.io_host)
+        default_ttl = 24 * 3600  # 1 day
+        region['ttl'] = region.get('ttl', default_ttl)
 
-        if len(hosts[self.scheme]) == 0 or self.io_host is None:
-            hosts = compat.json.loads(self.bucket_hosts(ak, bucket))
-        else:
-            # 1 year
-            hosts['ttl'] = int(time.time()) + 31536000
-        try:
-            scheme_hosts = hosts[self.scheme]
-        except KeyError:
-            raise KeyError(
-                "Please check your BUCKET_NAME! The UpHosts is %s" %
-                hosts)
         bucket_hosts = {
-            'upHosts': scheme_hosts['up'],
-            'ioHosts': scheme_hosts['io'],
-            'rsHosts': scheme_hosts['rs'],
-            'rsfHosts': scheme_hosts['rsf'],
-            'deadline': int(time.time()) + hosts['ttl']
+            'upHosts': [
+                '{0}://{1}'.format(self.scheme, domain)
+                for domain in region.get('up', {}).get('domains', [])
+            ],
+            'ioHosts': [
+                '{0}://{1}'.format(self.scheme, domain)
+                for domain in region.get('io', {}).get('domains', [])
+            ],
+            'rsHosts': [
+                '{0}://{1}'.format(self.scheme, domain)
+                for domain in region.get('rs', {}).get('domains', [])
+            ],
+            'rsfHosts': [
+                '{0}://{1}'.format(self.scheme, domain)
+                for domain in region.get('rsf', {}).get('domains', [])
+            ],
+            'apiHosts': [
+                '{0}://{1}'.format(self.scheme, domain)
+                for domain in region.get('api', {}).get('domains', [])
+            ],
+            'deadline': int(time.time()) + region['ttl']
         }
         home_dir = ""
         self.set_bucket_hosts_to_cache(key, bucket_hosts, home_dir)
@@ -166,9 +181,6 @@ def get_bucket_hosts_to_cache(self, key, home_dir):
         if len(self.host_cache) == 0:
             self.host_cache_from_file(home_dir)
 
-        if self.host_cache == {}:
-            return ret
-
         if key not in self.host_cache:
             return ret
 
@@ -211,7 +223,7 @@ def bucket_hosts(self, ak, bucket):
         uc_host = UC_HOST
         if is_customized_default('default_uc_host'):
             uc_host = get_default('default_uc_host')
-        url = "{0}/v1/query?ak={1}&bucket={2}".format(uc_host, ak, bucket)
+        url = "{0}/v4/query?ak={1}&bucket={2}".format(uc_host, ak, bucket)
         ret = requests.get(url)
         data = compat.json.dumps(ret.json(), separators=(',', ':'))
         return data

qiniu/services/storage/bucket.py

@@ -411,11 +411,7 @@ def bucket_domain(self, bucket_name):
         Args:
             bucket_name: 存储空间名
         """
-        options = {
-            'tbl': bucket_name,
-        }
-        url = "{0}/v6/domain/list?tbl={1}".format(config.get_default("default_api_host"), bucket_name)
-        return self.__get(url, options)
+        return self.list_domains(bucket_name)
 
     def change_bucket_permission(self, bucket_name, private):
         """
@@ -428,6 +424,12 @@ def change_bucket_permission(self, bucket_name, private):
         url = "{0}/private?bucket={1}&private={2}".format(config.get_default("default_uc_host"), bucket_name, private)
         return self.__post(url)
 
+    def __api_do(self, bucket, operation, data=None):
+        ak = self.auth.get_access_key()
+        api_host = self.zone.get_api_host(ak, bucket)
+        url = '{0}/{1}'.format(api_host, operation)
+        return self.__post(url, data)
+
     def __uc_do(self, operation, *args):
         return self.__server_do(config.get_default('default_uc_host'), operation, *args)