Skip to content

Commit 85f4339

Browse files
cornmonstercornmonster
authored
Merge pull request #24 from aliyun/release/0.33.x
Update to 0.33.7-public
2 parents 085e207 + 0e20c3b commit 85f4339

File tree

52 files changed

+1377
-234
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+1377
-234
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# 0.33.1
1+
# 0.33.7
22
- support list table
33
- table tunnel supports overwrite mode
44
- support global settings

‎odps-examples/tunnel-examples/src/main/java/TunnelComplexTypeSample.java‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
/**
2+
* Created by zhenhong.gzh on 16/7/13.
3+
*/
4+
5+
import java.io.IOException;
6+
import java.util.ArrayList;
7+
import java.util.Arrays;
8+
import java.util.HashMap;
9+
import java.util.List;
10+
import java.util.Map;
11+
12+
import com.aliyun.odps.Odps;
13+
import com.aliyun.odps.PartitionSpec;
14+
import com.aliyun.odps.TableSchema;
15+
import com.aliyun.odps.account.Account;
16+
import com.aliyun.odps.account.AliyunAccount;
17+
import com.aliyun.odps.data.ArrayRecord;
18+
import com.aliyun.odps.data.RecordReader;
19+
import com.aliyun.odps.data.RecordWriter;
20+
import com.aliyun.odps.data.SimpleStruct;
21+
import com.aliyun.odps.data.Struct;
22+
import com.aliyun.odps.tunnel.TableTunnel;
23+
import com.aliyun.odps.tunnel.TableTunnel.UploadSession;
24+
import com.aliyun.odps.tunnel.TableTunnel.DownloadSession;
25+
import com.aliyun.odps.tunnel.TunnelException;
26+
import com.aliyun.odps.type.StructTypeInfo;
27+
28+
public class TunnelComplexTypeSample {
29+
30+
private static String accessId = "<your access id>";
31+
private static String accessKey = "<your access Key>";
32+
private static String odpsUrl = "<your odps endpoint>";
33+
private static String project = "<your project>";
34+
35+
private static String table = "<your table name>";
36+
37+
// partitions of a partitioned table, eg: "pt=\'1\',ds=\'2\'"
38+
// if the table is not a partitioned table, do not need it
39+
private static String partition = "<your partition spec>";
40+
41+
public static void main(String args[]) {
42+
Account account = new AliyunAccount(accessId, accessKey);
43+
Odps odps = new Odps(account);
44+
odps.setEndpoint(odpsUrl);
45+
odps.setDefaultProject(project);
46+
47+
try {
48+
TableTunnel tunnel = new TableTunnel(odps);
49+
PartitionSpec partitionSpec = new PartitionSpec(partition);
50+
51+
// ---------- Upload Data ---------------
52+
// create upload session for table
53+
// the table schema is {"col0": ARRAY<BIGINT>, "col1": MAP<STRING, BIGINT>, "col2": STRUCT<name:STRING,age:BIGINT>}
54+
UploadSession uploadSession = tunnel.createUploadSession(project, table, partitionSpec);
55+
// get table schema
56+
TableSchema schema = uploadSession.getSchema();
57+
58+
// open record writer
59+
RecordWriter recordWriter = uploadSession.openRecordWriter(0);
60+
ArrayRecord record = (ArrayRecord) uploadSession.newRecord();
61+
62+
// prepare data
63+
List arrayData = Arrays.asList(1, 2, 3);
64+
Map<String, Long> mapData = new HashMap<String, Long>();
65+
mapData.put("a", 1L);
66+
mapData.put("c", 2L);
67+
68+
List<Object> structData = new ArrayList<Object>();
69+
structData.add("Lily");
70+
structData.add(18);
71+
72+
// set data to record
73+
record.setArray(0, arrayData);
74+
record.setMap(1, mapData);
75+
record.setStruct(2, new SimpleStruct((StructTypeInfo) schema.getColumn(2).getTypeInfo(),
76+
structData));
77+
78+
// write the record
79+
recordWriter.write(record);
80+
81+
// close writer
82+
recordWriter.close();
83+
84+
// commit uploadSession, the upload finish
85+
uploadSession.commit(new Long[]{0L});
86+
System.out.println("upload success!");
87+
88+
// ---------- Download Data ---------------
89+
// create download session for table
90+
// the table schema is {"col0": ARRAY<BIGINT>, "col1": MAP<STRING, BIGINT>, "col2": STRUCT<name:STRING,age:BIGINT>}
91+
DownloadSession downloadSession = tunnel.createDownloadSession(project, table, partitionSpec);
92+
schema = downloadSession.getSchema();
93+
94+
// open record reader, read one record here for example
95+
RecordReader recordReader = downloadSession.openRecordReader(0, 1);
96+
97+
// read the record
98+
ArrayRecord record1 = (ArrayRecord)recordReader.read();
99+
100+
// get array field data
101+
List field0 = record1.getArray(0);
102+
List<Long> longField0 = record1.getArray(Long.class, 0);
103+
104+
// get map field data
105+
Map field1 = record1.getMap(1);
106+
Map<String, Long> typedField1 = record1.getMap(String.class, Long.class, 1);
107+
108+
// get struct field data
109+
Struct field2 = record1.getStruct(2);
110+
111+
System.out.println("download success!");
112+
} catch (TunnelException e) {
113+
e.printStackTrace();
114+
} catch (IOException e) {
115+
e.printStackTrace();
116+
}
117+
118+
}
119+
}

‎odps-sdk-impl/odps-common-local/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
<parent>
66
<groupId>com.aliyun.odps</groupId>
77
<artifactId>odps-sdk-impl</artifactId>
8-
<version>0.32.5-public</version>
8+
<version>0.33.7-public</version>
99
</parent>
1010
<artifactId>odps-common-local</artifactId>
1111
<packaging>jar</packaging>

‎odps-sdk-impl/odps-graph-local/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
<parent>
88
<groupId>com.aliyun.odps</groupId>
99
<artifactId>odps-sdk-impl</artifactId>
10-
<version>0.32.5-public</version>
10+
<version>0.33.7-public</version>
1111
</parent>
1212

1313
<artifactId>odps-graph-local</artifactId>

‎odps-sdk-impl/odps-mapred-bridge/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
<parent>
88
<groupId>com.aliyun.odps</groupId>
99
<artifactId>odps-sdk-impl</artifactId>
10-
<version>0.32.5-public</version>
10+
<version>0.33.7-public</version>
1111
</parent>
1212

1313
<artifactId>odps-mapred-bridge</artifactId>

‎odps-sdk-impl/odps-mapred-bridge/src/main/java/com/aliyun/odps/mapred/BridgeJobRunner.java‎

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,10 @@
2323
import java.io.ByteArrayOutputStream;
2424
import java.io.File;
2525
import java.io.IOException;
26-
import java.net.MalformedURLException;
2726
import java.net.JarURLConnection;
27+
import java.net.MalformedURLException;
2828
import java.net.URL;
29+
import java.net.URLClassLoader;
2930
import java.util.ArrayList;
3031
import java.util.Collections;
3132
import java.util.HashMap;
@@ -37,10 +38,6 @@
3738
import java.util.jar.JarOutputStream;
3839
import java.util.jar.Manifest;
3940

40-
import com.google.gson.Gson;
41-
import com.google.gson.GsonBuilder;
42-
import com.google.gson.JsonParseException;
43-
import com.google.gson.reflect.TypeToken;
4441
import org.apache.commons.io.FilenameUtils;
4542
import org.apache.commons.lang.RandomStringUtils;
4643
import org.apache.commons.lang.StringUtils;
@@ -69,6 +66,10 @@
6966
import com.aliyun.odps.mapred.utils.InputUtils;
7067
import com.aliyun.odps.mapred.utils.OutputUtils;
7168
import com.aliyun.odps.mapred.utils.SchemaUtils;
69+
import com.google.gson.Gson;
70+
import com.google.gson.GsonBuilder;
71+
import com.google.gson.JsonParseException;
72+
import com.google.gson.reflect.TypeToken;
7273

7374
public abstract class BridgeJobRunner extends Configured implements JobRunner, EventListener {
7475

@@ -118,15 +119,15 @@ private void applyFrameworkResources() throws OdpsException {
118119

119120
Set<String> added = new HashSet<String>();
120121

121-
applyFrameworkResource(Odps.class, "odps-sdk-core.jar", padding, added);
122-
applyFrameworkResource(Mapper.class, "odps-sdk-mapred.jar", padding, added);
123-
applyFrameworkResource(BridgeJobRunner.class, "odps-mapred-bridge.jar",
122+
applyFrameworkResource(findJarFromClass(Odps.class), "odps-sdk-core.jar", padding, added);
123+
applyFrameworkResource(findJarFromClass(Mapper.class), "odps-sdk-mapred.jar", padding, added);
124+
applyFrameworkResource(findJarFromClass(BridgeJobRunner.class), "odps-mapred-bridge.jar",
124125
padding, added);
126+
125127
}
126128

127-
private void applyFrameworkResource(Class<?> clz, String alias,
128-
String padding, Set<String> added) throws OdpsException {
129-
String jarFilePath = "";
129+
private String findJarFromClass(Class<?> clz) throws OdpsException {
130+
String jarFilePath;
130131
try {
131132
URL jarUrl = clz.getProtectionDomain().getCodeSource().getLocation();
132133
String protocol = jarUrl.getProtocol();
@@ -141,7 +142,11 @@ private void applyFrameworkResource(Class<?> clz, String alias,
141142
} catch (Exception e) {
142143
throw new OdpsException("Get jar file path failed!", e);
143144
}
145+
return jarFilePath;
146+
}
144147

148+
private void applyFrameworkResource(String jarFilePath, String alias,
149+
String padding, Set<String> added) throws OdpsException {
145150
if (added.contains(jarFilePath)) {
146151
return;
147152
}
@@ -305,7 +310,9 @@ protected void setUp() throws OdpsException {
305310
new ByteArrayInputStream(jarOut.toByteArray()), jobId + ".jar", Resource.Type.JAR);
306311
aliasToTempResource.put("jobconf.jar", resName);
307312

308-
applyFrameworkResources();
313+
if (job.getBoolean("odps.mapred.upload.framework.resources.enable", true)) {
314+
applyFrameworkResources();
315+
}
309316

310317
List<String> totalRes = new ArrayList<String>();
311318
String[] resources = job.getResources();

‎odps-sdk-impl/odps-mapred-bridge/src/main/java/com/aliyun/odps/mapred/bridge/sqlgen/SqlGenerator.java‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,19 @@
1919

2020
package com.aliyun.odps.mapred.bridge.sqlgen;
2121

22+
import java.io.StringWriter;
23+
import java.util.Arrays;
24+
import java.util.HashSet;
25+
import java.util.Properties;
26+
import java.util.Set;
27+
28+
import org.apache.velocity.Template;
29+
import org.apache.velocity.VelocityContext;
30+
import org.apache.velocity.app.Velocity;
31+
import org.apache.velocity.exception.MethodInvocationException;
32+
import org.apache.velocity.exception.ParseErrorException;
33+
import org.apache.velocity.exception.ResourceNotFoundException;
34+
2235
import com.aliyun.odps.OdpsException;
2336
import com.aliyun.odps.data.VolumeInfo;
2437
import com.aliyun.odps.mapred.bridge.MetaExplorer;
@@ -28,18 +41,6 @@
2841
import com.aliyun.odps.mapred.utils.InputUtils;
2942
import com.aliyun.odps.mapred.utils.OutputUtils;
3043
import com.aliyun.odps.pipeline.Pipeline;
31-
import org.apache.velocity.Template;
32-
import org.apache.velocity.VelocityContext;
33-
import org.apache.velocity.app.Velocity;
34-
import org.apache.velocity.exception.MethodInvocationException;
35-
import org.apache.velocity.exception.ParseErrorException;
36-
import org.apache.velocity.exception.ResourceNotFoundException;
37-
38-
import java.io.StringWriter;
39-
import java.util.Arrays;
40-
import java.util.Set;
41-
import java.util.HashSet;
42-
import java.util.Properties;
4344

4445

4546
public class SqlGenerator {
@@ -50,7 +51,7 @@ public class SqlGenerator {
5051
aliasSets.add("odps-sdk-core.jar");
5152
aliasSets.add("odps-sdk-mapred.jar");
5253
aliasSets.add("odps-mapred-bridge.jar");
53-
// aliasSets.add("fastjson.jar");
54+
aliasSets.add("fastjson.jar");
5455
aliasSets.add("jobconf.jar");
5556
}
5657

‎odps-sdk-impl/odps-mapred-local/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
<parent>
88
<groupId>com.aliyun.odps</groupId>
99
<artifactId>odps-sdk-impl</artifactId>
10-
<version>0.32.5-public</version>
10+
<version>0.33.7-public</version>
1111
</parent>
1212

1313
<artifactId>odps-mapred-local</artifactId>

‎odps-sdk-impl/odps-udf-example/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
<parent>
88
<groupId>com.aliyun.odps</groupId>
99
<artifactId>odps-sdk-impl</artifactId>
10-
<version>0.32.5-public</version>
10+
<version>0.33.7-public</version>
1111
</parent>
1212

1313
<artifactId>odps-udf-example</artifactId>

‎odps-sdk-impl/odps-udf-local/pom.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
<parent>
66
<groupId>com.aliyun.odps</groupId>
77
<artifactId>odps-sdk-impl</artifactId>
8-
<version>0.32.5-public</version>
8+
<version>0.33.7-public</version>
99
</parent>
1010
<artifactId>odps-udf-local</artifactId>
1111
<packaging>jar</packaging>
@@ -31,7 +31,7 @@
3131
<dependency>
3232
<groupId>com.aliyun.odps</groupId>
3333
<artifactId>odps-udf-example</artifactId>
34-
<version>0.32.5-public</version>
34+
<version>0.33.7-public</version>
3535
</dependency>
3636
</dependencies>
3737

0 commit comments

Comments
 (0)