Skip to content

Commit 3d95fa6

Browse files
cornmonstercornmonster
authored
Merge pull request #33 from aliyun/release/0.36.4-public
Update to 0.36.4-public
2 parents 889feed + 074bd45 commit 3d95fa6

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+1688
-321
lines changed

‎odps-sdk-impl/odps-mapred-bridge/src/main/java/com/aliyun/odps/mapred/cli/OdpsConf.java‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ public class OdpsConf extends Configuration {
4545
private final static String PROJNAME = "odps.project.name";
4646
private final static String ENDPOINT = "odps.end.point";
4747
private final static String ACCESSID = "odps.access.id";
48+
private final static String STS_TOKEN = "odps.sts.token";
4849
private final static String ACCESSKEY = "odps.access.key";
4950
private final static String ACCOUNT_PROVIDER = "odps.account.provider";
5051

@@ -132,6 +133,21 @@ public void setAccessKey(String accessKey) {
132133
set(ACCESSKEY, accessKey);
133134
}
134135

136+
/**
137+
* 获取STS token
138+
* @return STS token
139+
*/
140+
public String getStsToken() {
141+
return get(STS_TOKEN);
142+
}
143+
144+
/**
145+
* 设置STS token
146+
*/
147+
public void setStsToken(String stsToken) {
148+
set(STS_TOKEN, stsToken);
149+
}
150+
135151
/**
136152
* 获取代理主机地址.
137153
*
@@ -144,7 +160,7 @@ public String getProxyHost() {
144160
/**
145161
* 设置代理主机地址.
146162
*
147-
* @param proxyHost
163+
* @param host
148164
*/
149165
public void setProxyHost(String host) {
150166
set(PROXYHOST, host);
@@ -180,7 +196,7 @@ public String getProxyPort() {
180196
/**
181197
* 设置代理端口.
182198
*
183-
* @param proxyPort
199+
* @param port
184200
*/
185201
public void setProxyPort(String port) {
186202
set(PROXYPORT, port);
@@ -190,12 +206,11 @@ public void setProxyPort(String port) {
190206
* 获取帐号类型<br/>
191207
* 取值范围：{@link com.aliyun.odps.account.Account.AccountProvider#ALIYUN} ,
192208
* {@link com.aliyun.odps.account.Account.AccountProvider#TAOBAO},
193-
* {@link com.aliyun.odps.account.Account.AccountProvider#DOMAIN} <br/>
209+
* {@link com.aliyun.odps.account.Account.AccountProvider#BEARER_TOKEN}
210+
* {@link com.aliyun.odps.account.Account.AccountProvider#STS}<br/>
194211
* 默认值： {@link com.aliyun.odps.account.Account.AccountProvider#ALIYUN}
195212
*
196-
* @return 帐号类型：{@link com.aliyun.odps.account.Account.AccountProvider#ALIYUN}
197-
* |{@link com.aliyun.odps.account.Account.AccountProvider#TAOBAO}|
198-
* {@link com.aliyun.odps.account.Account.AccountProvider#DOMAIN}
213+
* @return 帐号类型
199214
* @throws OdpsException
200215
*/
201216
public AccountProvider getAccountProvider() throws OdpsException {
@@ -206,7 +221,7 @@ public AccountProvider getAccountProvider() throws OdpsException {
206221
try {
207222
result = AccountProvider.valueOf(provider);
208223
} catch (Exception exception) {
209-
throw new OdpsException("Unsupport account provider:" + provider);
224+
throw new OdpsException("Unsupported account provider:" + provider);
210225
}
211226
}
212227
return result;

‎odps-sdk-impl/odps-mapred-bridge/src/main/java/com/aliyun/odps/mapred/cli/OptionParser.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import java.util.Arrays;
2727
import java.util.Map;
2828

29+
import com.aliyun.odps.account.StsAccount;
2930
import com.google.gson.Gson;
3031
import com.google.gson.GsonBuilder;
3132
import com.google.gson.reflect.TypeToken;
@@ -197,6 +198,10 @@ private Account getAccount(OdpsConf odpsConf) throws OdpsException {
197198
return new AliyunAccount(odpsConf.getAccessId(), odpsConf.getAccessKey());
198199

199200

201+
case STS:
202+
return new StsAccount(odpsConf.getAccessId(),
203+
odpsConf.getAccessKey(),
204+
odpsConf.getStsToken());
200205
default:
201206
throw new OdpsException("unsupport account provider:" + accountProvider);
202207
}

‎odps-sdk/odps-sdk-commons/src/main/java/com/aliyun/odps/data/ArrayRecord.java‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
* @see Record
4040
*/
4141
public class ArrayRecord implements Record {
42+
private static final Long DEFAULT_FIELD_MAX_SIZE = 8 * 1024 * 1024L;
4243
private static final String DEFAULT_CHARSET = "utf-8";
4344

4445
private Column[] columns;
@@ -50,26 +51,35 @@ public class ArrayRecord implements Record {
5051
* restricted.
5152
*/
5253
private boolean strictTypeValidation;
54+
/**
55+
* Field max size allowed.
56+
*/
57+
private Long fieldMaxSize = DEFAULT_FIELD_MAX_SIZE;
5358

5459
public ArrayRecord(Column[] columns) {
5560
this(columns, true);
5661
}
5762

5863
public ArrayRecord(Column[] columns, boolean strictTypeValidation) {
64+
this(columns, strictTypeValidation, DEFAULT_FIELD_MAX_SIZE);
65+
}
5966

67+
public ArrayRecord(Column[] columns, boolean strictTypeValidation, Long fieldMaxSize) {
6068
if (columns == null) {
6169
throw new IllegalArgumentException();
6270
}
6371

6472
this.columns = columns;
6573
this.strictTypeValidation = strictTypeValidation;
74+
if (fieldMaxSize != null) {
75+
this.fieldMaxSize = fieldMaxSize;
76+
}
6677

6778
values = new Object[columns.length];
6879

6980
for (int i = 0; i < columns.length; i++) {
7081
nameMap.put(columns[i].getName(), i);
7182
}
72-
7383
}
7484

7585
public ArrayRecord(Column[] columns, Object[] values) {
@@ -113,8 +123,11 @@ public void set(int idx, Object value) {
113123
return;
114124
}
115125

116-
values[idx] =
117-
OdpsTypeTransformer.transform(value, columns[idx].getTypeInfo(), strictTypeValidation);
126+
values[idx] = OdpsTypeTransformer.transform(
127+
value,
128+
columns[idx].getTypeInfo(),
129+
strictTypeValidation,
130+
fieldMaxSize);
118131
}
119132

120133
@Override

‎odps-sdk/odps-sdk-commons/src/main/java/com/aliyun/odps/data/OdpsTypeTransformer.java‎

Lines changed: 31 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -22,17 +22,16 @@
2222
* Created by zhenhong.gzh on 16/12/13.
2323
*/
2424
public class OdpsTypeTransformer {
25-
private static final int STRING_MAX_LENGTH = 8 * 1024 * 1024;
25+
private static final int UTF8_ENCODED_CHAR_MAX_SIZE = 6;
2626

2727
// 9999-12-31 23:59:59
2828
private static final long DATETIME_MAX_TICKS = 253402271999000L;
2929
// 0001-01-01 00:00:00
3030
private static final long DATETIME_MIN_TICKS = -62135798400000L;
3131

32-
private static Map<OdpsType, Class> ODPS_TYPE_MAPPER = new HashMap<OdpsType, Class>() ;
32+
private static Map<OdpsType, Class> ODPS_TYPE_MAPPER = new HashMap<OdpsType, Class>();
3333

3434
OdpsTypeTransformer() {
35-
3635
}
3736

3837
static {
@@ -66,12 +65,12 @@ public static Class odpsTypeToJavaType(OdpsType type) {
6665
throw new IllegalArgumentException("Cannot get Java type for Odps type: " + type);
6766
}
6867

69-
private static void validateString(String value) {
68+
private static void validateString(String value, long limit) {
7069
try {
71-
if ((value.length() * 6 > STRING_MAX_LENGTH) && (value.getBytes("utf-8").length
72-
> STRING_MAX_LENGTH)) {
73-
throw new IllegalArgumentException("InvalidData: The string's length is more than "
74-
+ STRING_MAX_LENGTH / 1024 / 1024 + "M.");
70+
if (value.length() * UTF8_ENCODED_CHAR_MAX_SIZE > limit
71+
&& value.getBytes("utf-8").length > limit) {
72+
throw new IllegalArgumentException(
73+
"InvalidData: The string's length is more than " + limit + " bytes.");
7574
}
7675
} catch (UnsupportedEncodingException e) {
7776
throw new IllegalArgumentException(e.getMessage(), e);
@@ -117,19 +116,27 @@ private static void validateDecimal(BigDecimal value, DecimalTypeInfo typeInfo)
117116

118117
}
119118

120-
private static List transformArray(List value, ArrayTypeInfo typeInfo) {
119+
private static List transformArray(
120+
List value,
121+
ArrayTypeInfo typeInfo,
122+
boolean strict,
123+
long fieldMaxSize) {
121124
List<Object> newList = new ArrayList<Object>(value.size());
122125

123126
TypeInfo elementTypeInfo = typeInfo.getElementTypeInfo();
124127

125128
for (Object obj : value) {
126-
newList.add(transform(obj, elementTypeInfo));
129+
newList.add(transform(obj, elementTypeInfo, strict, fieldMaxSize));
127130
}
128131

129132
return newList;
130133
}
131134

132-
private static Map transformMap(Map value, MapTypeInfo typeInfo) {
135+
private static Map transformMap(
136+
Map value,
137+
MapTypeInfo typeInfo,
138+
boolean strict,
139+
long fieldMaxSize) {
133140
TypeInfo keyTypeInfo = typeInfo.getKeyTypeInfo();
134141
TypeInfo valTypeInfo = typeInfo.getValueTypeInfo();
135142

@@ -139,31 +146,31 @@ private static Map transformMap(Map value, MapTypeInfo typeInfo) {
139146
while (iter.hasNext()) {
140147
Map.Entry entry = (Map.Entry) iter.next();
141148

142-
Object entryKey = transform(entry.getKey(), keyTypeInfo);
143-
Object entryValue = transform(entry.getValue(), valTypeInfo);
149+
Object entryKey = transform(entry.getKey(), keyTypeInfo, strict, fieldMaxSize);
150+
Object entryValue = transform(entry.getValue(), valTypeInfo, strict, fieldMaxSize);
144151

145152
newMap.put(entryKey, entryValue);
146153
}
147154

148155
return newMap;
149156
}
150157

151-
private static Struct transformStruct(Struct value, StructTypeInfo typeInfo) {
158+
private static Struct transformStruct(
159+
Struct value,
160+
StructTypeInfo typeInfo,
161+
boolean strict,
162+
long fieldMaxSize) {
152163
List<Object> elements = new ArrayList<Object>();
153164

154165
for (int i = 0; i < typeInfo.getFieldCount(); ++i) {
155166
TypeInfo fieldTypeInfo = value.getFieldTypeInfo(i);
156-
elements.add(transform(value.getFieldValue(i), fieldTypeInfo));
167+
elements.add(transform(value.getFieldValue(i), fieldTypeInfo, strict, fieldMaxSize));
157168
}
158169

159170
return new SimpleStruct(typeInfo, elements);
160171
}
161172

162-
static Object transform(Object value, TypeInfo typeInfo) {
163-
return transform(value, typeInfo, true);
164-
}
165-
166-
static Object transform(Object value, TypeInfo typeInfo, boolean strict) {
173+
static Object transform(Object value, TypeInfo typeInfo, boolean strict, long fieldMaxSize) {
167174
if (value == null) {
168175
return null;
169176
}
@@ -176,7 +183,7 @@ static Object transform(Object value, TypeInfo typeInfo, boolean strict) {
176183
value = ArrayRecord.bytesToString((byte []) value);
177184
}
178185
if (strict) {
179-
validateString((String) value);
186+
validateString((String) value, fieldMaxSize);
180187
}
181188
break;
182189
case BIGINT:
@@ -197,11 +204,11 @@ static Object transform(Object value, TypeInfo typeInfo, boolean strict) {
197204
validateVarChar((Varchar) value, (VarcharTypeInfo) typeInfo);
198205
break;
199206
case ARRAY:
200-
return transformArray((List) value, (ArrayTypeInfo) typeInfo);
207+
return transformArray((List) value, (ArrayTypeInfo) typeInfo, strict, fieldMaxSize);
201208
case MAP:
202-
return transformMap((Map) value, (MapTypeInfo) typeInfo);
209+
return transformMap((Map) value, (MapTypeInfo) typeInfo, strict, fieldMaxSize);
203210
case STRUCT:
204-
return transformStruct((Struct) value, (StructTypeInfo) typeInfo);
211+
return transformStruct((Struct) value, (StructTypeInfo) typeInfo, strict, fieldMaxSize);
205212
default:
206213
}
207214

‎odps-sdk/odps-sdk-core/pom.xml‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,12 @@
3030
<groupId>org.codehaus.jackson</groupId>
3131
<artifactId>jackson-mapper-asl</artifactId>
3232
</dependency>
33-
33+
34+
<dependency>
35+
<groupId>com.google.guava</groupId>
36+
<artifactId>guava</artifactId>
37+
</dependency>
38+
3439
<dependency>
3540
<groupId>com.google.code.gson</groupId>
3641
<artifactId>gson</artifactId>
@@ -142,4 +147,3 @@
142147
</plugins>
143148
</reporting>
144149
</project>
145-

‎odps-sdk/odps-sdk-core/src/main/java/com/aliyun/odps/Instance.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -489,20 +489,20 @@ public TaskCost getTaskCost(String taskName) throws OdpsException {
489489

490490
try {
491491
if (summary.get("Cost") != null) {
492-
Map<String, Integer> taskCostMap = (Map) summary.get("Cost");
492+
Map<String, Long> taskCostMap = (Map) summary.get("Cost");
493493

494494
TaskCost cost = new TaskCost();
495495

496496
if (taskCostMap.get("CPU") != null) {
497-
cost.setCPUCost(taskCostMap.get("CPU"));
497+
cost.setCPUCost(taskCostMap.get("CPU").intValue());
498498
}
499499

500500
if (taskCostMap.get("Memory") != null) {
501-
cost.setMemoryCost(taskCostMap.get("Memory"));
501+
cost.setMemoryCost(taskCostMap.get("Memory").intValue());
502502
}
503503

504504
if (taskCostMap.get("Input") != null) {
505-
cost.setInputSize(taskCostMap.get("Input"));
505+
cost.setInputSize(taskCostMap.get("Input").intValue());
506506
}
507507

508508
return cost;

‎odps-sdk/odps-sdk-core/src/main/java/com/aliyun/odps/Odps.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525

2626
import com.aliyun.odps.account.Account;
2727
import com.aliyun.odps.account.AccountFormat;
28+
import com.aliyun.odps.account.AppStsAccount;
2829
import com.aliyun.odps.commons.transport.DefaultTransport;
2930
import com.aliyun.odps.ml.OfflineModels;
3031
import com.aliyun.odps.rest.RestClient;
@@ -69,6 +70,7 @@ public class Odps {
6970

7071
private Account account;
7172
private AppAccount appAccount;
73+
private AppStsAccount appStsAccount;
7274
private String endpoint;
7375
private String defaultProject;
7476

@@ -100,6 +102,11 @@ public void setAppAccount(AppAccount appAccount) {
100102
this.appAccount = appAccount;
101103
}
102104

105+
public void setAppStsAccount(AppStsAccount appStsAccount) {
106+
this.client.setAppStsAccount(appStsAccount);
107+
this.appStsAccount = appStsAccount;
108+
}
109+
103110
/**
104111
* 指定{@link Account}构造Odps对象
105112
*
@@ -212,6 +219,14 @@ public AppAccount getAppAccount() {
212219
return appAccount;
213220
}
214221

222+
/**
223+
* 获取{@link AppStsAccount}
224+
* @return {@link AppStsAccount}
225+
*/
226+
public AppStsAccount getAppStsAccount() {
227+
return appStsAccount;
228+
}
229+
215230
/**
216231
* 获取ODPS服务的地址
217232
*

0 commit comments

Comments
 (0)