Skip to content

Commit 0b9a8d2

Browse files
dingxin-techdingxin-tech
committed
prepare for 0.51.8-public
1 parent 80560f5 commit 0b9a8d2

File tree

9 files changed

+249
-40
lines changed

9 files changed

+249
-40
lines changed

‎docs/docs/api-reference/Partition.md‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
---
2+
title: Partition
3+
sidebar_position: 4
4+
---
5+
6+
# Partition
7+
8+
`Partition` 类代表 ODPS 中的分区，可以用于进行分区基本信息的查询操作。
9+
10+
## 概览
11+
12+
- [获取分区实例对象](#获取分区实例对象)
13+
- [分区基本信息](#分区基本信息)
14+
- [分区扩展信息](#分区扩展信息)
15+
16+
## 获取分区实例对象
17+
18+
要操作分区，首先需要创获取一个分区的实例对象。
19+
20+
> 注意，获取分区实例是一个lazy操作，即只有当调用 `Partition` 类的其他方法时，才会真正获取元数据信息。同时，只有分区真实存在，才能获取到分区实例。
21+
22+
```java
23+
Partition partition = table.get(new PartitionSpec("partition_spec"));
24+
```
25+
## 分区基本信息
26+
`Partition` 作为一个实现 `lazyload`功能的对象，只有在调用以下方法时，才会进行网络通信，获取真正的元数据信息。
27+
同时，我们也可以手动调用`reload()`方法，来达到同样的效果。
28+
29+
```java
30+
partitoin.reload();
31+
```
32+
除非手动触发`reload()`方法，否则 Partition 在`reload`一次之后默认使用缓存。
33+
34+
35+
### 获取分区
36+
37+
```java
38+
PartitionSpec = partition.getPartitionSpec();
39+
```
40+
41+
### 获取创建时间
42+
43+
```java
44+
Date createdTime = partition.getCreatedTime();
45+
```
46+
47+
### 获取最后修改时间
48+
49+
```java
50+
Date lastMetaModifiedTime = partition.getLastMetaModifiedTime();
51+
```
52+
53+
### 获取存储大小
54+
注意：此方法通常不保证和实际占用存储大小相同。单位为`bytes`
55+
```java
56+
long size = partition.getSize();
57+
```
58+
59+
### 获取数据数量
60+
注意：此方法当无准确数据时，返回-1
61+
```java
62+
long recordNum = partition.getRecordNum();
63+
```
64+
65+
### 获取生命周期
66+
单位为天
67+
```java
68+
long lifeCycle = partition.getLifeCycle();
69+
```
70+
## 分区扩展信息
71+
这部分信息也属于`lazyload`操作，会在第一次调用此类方法时获取数据，然而，无法通过手动调用`reload()`方法刷新。
72+
因为这部分信息通常随着分区的创建而确定，或变化较少。
73+
74+
### 查看扩展信息的保留字段
75+
返回值为Json字符串
76+
```java
77+
String reserved = partition.getReserved();
78+
```
79+
80+
### 查看分区是否进行过归档（archive）操作
81+
返回true表示进行过归档（archive）操作，false表示未进行过
82+
```java
83+
partition.isArchived();
84+
```
85+
86+
### 查看分区所占磁盘的物理大小
87+
注意：此类方法通常为估计值，不保证准确性
88+
```java
89+
long physicalSize = partition.getPhysicalSize();
90+
```
91+
92+
### 查看分区占用文件数
93+
注意：此类方法通常为估计值，不保证准确性
94+
```java
95+
long fileNum = partition.getFileNum();
96+
```
97+
98+
### 查看 Cluster 信息
99+
```java
100+
ClusterInfo clusterInfo = partition.getClusterInfo();
101+
```
102+
103+
### 查看 CDC 信息
104+
105+
```java
106+
long cdcSize = partition.getCdcSize();
107+
long cdcRecordNum = partition.getCdcRecordNum();
108+
```
109+

‎odps-sdk/odps-sdk-commons/src/main/java/com/aliyun/odps/PartitionSpec.java‎

Lines changed: 24 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -54,23 +54,36 @@ public PartitionSpec(String spec) {
5454

5555
public PartitionSpec(String spec, boolean trim) {
5656
if (spec == null) {
57-
throw new IllegalArgumentException("Argument \'spec\' cannot be null");
57+
throw new IllegalArgumentException("Argument 'spec' cannot be null");
5858
}
5959
String[] groups = spec.split("[,/]");
6060
for (String group : groups) {
61-
String[] kv = group.split("=");
62-
if (kv.length != 2) {
63-
throw new IllegalArgumentException("Invalid partition spec.");
61+
if (group.isEmpty()) {
62+
throw new IllegalArgumentException(
63+
String.format("Invalid partition spec: empty group in '%s'", spec)
64+
);
6465
}
65-
66-
String k = trim ? kv[0].trim() : kv[0];
67-
String v = (trim ? kv[1].trim() : kv[1])
68-
.replaceAll("'", "")
69-
.replaceAll("\"", "");
70-
if (k.length() == 0 || v.length() == 0) {
71-
throw new IllegalArgumentException("Invalid partition spec.");
66+
String[] splits = group.split("=", 2);
67+
if (splits.length != 2) {
68+
throw new IllegalArgumentException(
69+
String.format("Invalid partition spec: expected key=value in '%s'", group)
70+
);
7271
}
7372

73+
String k = trim ? splits[0].trim() : splits[0];
74+
String v = (trim ? splits[1].trim() : splits[1])
75+
.replace("'", "")
76+
.replace("\"", "");
77+
if (k.isEmpty()) {
78+
throw new IllegalArgumentException(
79+
String.format("Invalid partition spec: empty key in group '%s'", group)
80+
);
81+
}
82+
if (v.isEmpty()) {
83+
throw new IllegalArgumentException(
84+
String.format("Invalid partition spec: empty value for key '%s' in group '%s'", k, group)
85+
);
86+
}
7487
set(k, v);
7588
}
7689
}

‎odps-sdk/odps-sdk-commons/src/main/java/com/aliyun/odps/data/ArrayRecord.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ public ArrayRecord(Column[] columns, boolean strictTypeValidation, Long fieldMax
9595
values = new Serializable[columns.length];
9696

9797
for (int i = 0; i < columns.length; i++) {
98-
nameMap.put(columns[i].getName(), i);
98+
nameMap.put(columns[i].getName().toLowerCase(), i);
9999
}
100100
}
101101

@@ -402,7 +402,7 @@ public Object[] toArray() {
402402
}
403403

404404
private int getColumnIndex(String name) {
405-
Integer idx = nameMap.get(name);
405+
Integer idx = nameMap.get(name.toLowerCase());
406406
if (idx == null) {
407407
throw new IllegalArgumentException("No such column:" + name);
408408
}

‎odps-sdk/odps-sdk-core/src/main/java/com/aliyun/odps/Partition.java‎

Lines changed: 53 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,9 @@ static class PartitionModel {
7878
* 分层存储相关
7979
*/
8080
StorageTierInfo storageTierInfo;
81+
82+
long cdcSize = -1;
83+
long cdcRecordNum = -1;
8184
}
8285

8386
@Root(name = "Column", strict = false)
@@ -118,6 +121,10 @@ static class PartitionSpecModel {
118121
private String reserved;
119122
private Table.ClusterInfo clusterInfo;
120123

124+
// raw json info
125+
private String metadataJson;
126+
private String extendedInfoJson;
127+
121128
Partition(
122129
PartitionModel model,
123130
String projectName,
@@ -237,6 +244,26 @@ public long getRecordNum() {
237244
return recordNum;
238245
}
239246

247+
/**
248+
* 获取分区内部存储的大小，单位Byte
249+
*
250+
* @return 内部存储大小
251+
*/
252+
public long getSize() {
253+
lazyLoad();
254+
return size;
255+
}
256+
257+
/**
258+
* Get metadata json string.
259+
*
260+
* @return metadata json string
261+
*/
262+
public String getMetadataJson() {
263+
lazyLoad();
264+
return metadataJson;
265+
}
266+
240267
/**
241268
* 查看表是否进行了归档
242269
*
@@ -282,26 +309,13 @@ public long getFileNum() {
282309
return fileNum;
283310
}
284311

285-
/**
286-
* 获取分区内部存储的大小，单位Byte
287-
*
288-
* @return 内部存储大小
289-
*/
290-
public long getSize() {
291-
lazyLoad();
292-
return size;
293-
}
294-
295312
/**
296313
* 得到分区的存储类型
297314
*
298315
* @return StorageTierType
299316
*/
300317
public StorageTierInfo getStorageTierInfo() {
301-
if (model.storageTierInfo == null) {
302-
reloadExtendInfo();
303-
isExtendInfoLoaded = true;
304-
}
318+
lazyLoadExtendInfo();
305319
return model.storageTierInfo;
306320
}
307321

@@ -312,9 +326,7 @@ public StorageTierInfo getStorageTierInfo() {
312326
* @return 保留字段
313327
*/
314328
public String getReserved() {
315-
if (reserved == null) {
316-
lazyLoadExtendInfo();
317-
}
329+
lazyLoadExtendInfo();
318330
return reserved;
319331
}
320332

@@ -324,13 +336,25 @@ public String getReserved() {
324336
* @return cluster info
325337
*/
326338
public Table.ClusterInfo getClusterInfo() {
327-
if (clusterInfo == null) {
328-
lazyLoadExtendInfo();
329-
}
330-
339+
lazyLoadExtendInfo();
331340
return clusterInfo;
332341
}
333342

343+
public long getCdcSize() {
344+
lazyLoadExtendInfo();
345+
return model.cdcSize;
346+
}
347+
348+
public long getCdcRecordNum() {
349+
lazyLoadExtendInfo();
350+
return model.cdcRecordNum;
351+
}
352+
353+
public String getExtendedInfoJson() {
354+
lazyLoadExtendInfo();
355+
return extendedInfoJson;
356+
}
357+
334358
/**
335359
* Get {@link Tag}(s) attached to this partition.
336360
*
@@ -551,6 +575,7 @@ public void reload() throws OdpsException {
551575
params.put("partition", getPartitionSpec().toString());
552576

553577
PartitionMeta meta = client.request(PartitionMeta.class, resource, "GET", params);
578+
metadataJson = meta.schema;
554579

555580
try {
556581
JsonObject tree = new JsonParser().parse(meta.schema).getAsJsonObject();
@@ -606,6 +631,7 @@ public void reloadExtendInfo() {
606631
PartitionMeta meta =
607632
client.request(PartitionMeta.class, resource.toString(), "GET", params);
608633

634+
extendedInfoJson = meta.schema;
609635
JsonObject tree = new JsonParser().parse(meta.schema).getAsJsonObject();
610636

611637
if (tree.has("IsArchived")) {
@@ -651,5 +677,11 @@ private void loadReservedJson(String reserved) {
651677
clusterInfo = Table.parseClusterInfo(reservedJson);
652678
// 分层存储的相关信息
653679
model.storageTierInfo = StorageTierInfo.getStorageTierInfo(reservedJson);
680+
681+
model.cdcSize = reservedJson.has("cdc_size") ? Long.parseLong(
682+
reservedJson.get("cdc_size").getAsString()) : -1;
683+
684+
model.cdcRecordNum = reservedJson.has("cdc_record_num") ? Long.parseLong(
685+
reservedJson.get("cdc_record_num").getAsString()) : -1;
654686
}
655687
}

‎odps-sdk/odps-sdk-core/src/main/java/com/aliyun/odps/Table.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -397,6 +397,10 @@ public String toStringWithQuote() {
397397
private boolean isShardInfoLoaded;
398398
private Odps odps;
399399

400+
// Raw Json metadata
401+
private String metadataJson;
402+
private String extendedInfoJson;
403+
400404
Table(TableModel model, String project, String schemaName, Odps odps) {
401405
this.model = model;
402406
this.model.projectName = project;
@@ -423,6 +427,7 @@ public void reload() throws OdpsException {
423427
String resource = ResourceBuilder.buildTableResource(model.projectName, model.name);
424428
Map<String, String> params = initParamsWithSchema();
425429
reload(client.request(TableModel.class, resource, "GET", params));
430+
this.metadataJson = model.schema.content;
426431
return null;
427432
}
428433
);
@@ -458,6 +463,7 @@ public void reloadExtendInfo() {
458463
} catch (OdpsException e) {
459464
throw new ReloadException(e.getMessage(), e);
460465
}
466+
this.extendedInfoJson = response.schema.content;
461467
loadSchemaFromJson(response.schema.content);
462468
}
463469

@@ -1145,6 +1151,16 @@ public String getJsonSchema() {
11451151
return model.schema == null ? null : model.schema.content;
11461152
}
11471153

1154+
public String getMetadataJson() {
1155+
lazyLoad();
1156+
return metadataJson;
1157+
}
1158+
1159+
public String getExtendedInfoJson() {
1160+
lazyLoadExtendInfo();
1161+
return extendedInfoJson;
1162+
}
1163+
11481164
/**
11491165
* 查看表是否进行过归档操作
11501166
*

0 commit comments

Comments
 (0)