Skip to content

Commit 5c0919b

Browse files
vidmavidma
committed
Update
1 parent 0fd7f4f commit 5c0919b

File tree

6 files changed

+293
-26
lines changed

6 files changed

+293
-26
lines changed

‎simple-spark-sources/build.sbt

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11

2-
organization := "generated"
2+
organization := "com.example"
33

44
name := "simple-spark"
55

@@ -9,7 +9,7 @@ crossPaths := true
99

1010
scalaVersion := "2.10.6"
1111

12-
maintainer := "" //Docker
12+
maintainer := "DF" //Docker
1313

1414
resolvers ++= Seq(
1515
"Maven2 Local" at "file:/Users/vidma-vinted/.m2/repository/" ,
@@ -42,7 +42,7 @@ dockerExposedPorts := Seq(9000, 9443)
4242

4343
daemonUser in Docker := "root"
4444

45-
packageName in Docker := "generated.simplespark"
45+
packageName in Docker := "com.example.simplespark"
4646

4747
mappings in Docker ++= directory("spark-lib")
4848

@@ -152,7 +152,7 @@ libraryDependencies += "com.google.guava" % "guava" % "16.0.1" force()
152152
test in assembly := {}
153153

154154
//main class
155-
mainClass in assembly := Some("generated.Main")
155+
mainClass in assembly := Some("com.example.Main")
156156

157157
artifact in (Compile, assembly) ~= { art =>
158158
art.copy(`classifier` = Some("assembly"))

‎simple-spark-sources/simple-spark.snb.ipynb

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
{
2+
"metadata" : {
3+
"id" : "af838c83-8642-46b6-9358-12ee05e704e1",
4+
"name" : "Simple Spark",
5+
"user_save_timestamp" : "2014-10-11T17:33:45.703Z",
6+
"auto_save_timestamp" : "2015-01-10T00:02:12.659Z",
7+
"language_info" : {
8+
"name" : "scala",
9+
"file_extension" : "scala",
10+
"codemirror_mode" : "text/x-scala"
11+
},
12+
"trusted" : true,
13+
"sparkNotebook" : null,
14+
"customLocalRepo" : null,
15+
"customRepos" : null,
16+
"customDeps" : null,
17+
"customImports" : null,
18+
"customArgs" : null,
19+
"customSparkConf" : null,
20+
"customVars" : null
21+
},
22+
"cells" : [ {
23+
"metadata" : {
24+
"id" : "37DEEDC733044DD9972E9E4BA2F1B126"
25+
},
26+
"cell_type" : "markdown",
27+
"source" : "### Spark config"
28+
}, {
29+
"metadata" : {
30+
"trusted" : true,
31+
"input_collapsed" : false,
32+
"collapsed" : true,
33+
"id" : "2A2F6AA618AC48018D01E7D2F4183B76"
34+
},
35+
"cell_type" : "code",
36+
"source" : "sparkContext.getConf.toDebugString",
37+
"outputs" : [ ]
38+
}, {
39+
"metadata" : {
40+
"id" : "DAFA77C3B6D140FF8AAE30B94D2FC73E"
41+
},
42+
"cell_type" : "markdown",
43+
"source" : "#### Counting"
44+
}, {
45+
"metadata" : {
46+
"trusted" : true,
47+
"input_collapsed" : false,
48+
"collapsed" : false,
49+
"id" : "9088B578DE2F4BA48DF323F11895488A"
50+
},
51+
"cell_type" : "code",
52+
"source" : "def transform(i: Int) = (i, i+1)",
53+
"outputs" : [ {
54+
"name" : "stdout",
55+
"output_type" : "stream",
56+
"text" : "transform: (i: Int)(Int, Int)\n"
57+
}, {
58+
"metadata" : { },
59+
"data" : {
60+
"text/html" : ""
61+
},
62+
"output_type" : "execute_result",
63+
"execution_count" : 2,
64+
"time" : "Took: 0.727s, at 2017-05-16 12:18"
65+
} ]
66+
}, {
67+
"metadata" : {
68+
"trusted" : true,
69+
"input_collapsed" : false,
70+
"collapsed" : false,
71+
"presentation" : {
72+
"tabs_state" : "{\n \"tab_id\": \"#tab662761867-0\"\n}",
73+
"pivot_chart_state" : "{\n \"hiddenAttributes\": [],\n \"menuLimit\": 200,\n \"cols\": [],\n \"rows\": [],\n \"vals\": [],\n \"exclusions\": {},\n \"inclusions\": {},\n \"unusedAttrsVertical\": 85,\n \"autoSortUnusedAttrs\": false,\n \"inclusionsInfo\": {},\n \"aggregatorName\": \"Count\",\n \"rendererName\": \"Table\"\n}"
74+
},
75+
"id" : "BF434E47187740E78B7A7A521D2D87DD"
76+
},
77+
"cell_type" : "code",
78+
"source" : "val dataset = sparkSession.createDataset(1 to 1000).map(transform)",
79+
"outputs" : [ {
80+
"name" : "stdout",
81+
"output_type" : "stream",
82+
"text" : "dataset: org.apache.spark.sql.Dataset[(Int, Int)] = [_1: int, _2: int]\n"
83+
}, {
84+
"metadata" : { },
85+
"data" : {
86+
"text/html" : ""
87+
},
88+
"output_type" : "execute_result",
89+
"execution_count" : 9,
90+
"time" : "Took: 1.466s, at 2017-05-16 12:20"
91+
} ]
92+
}, {
93+
"metadata" : {
94+
"trusted" : true,
95+
"input_collapsed" : false,
96+
"collapsed" : false,
97+
"presentation" : {
98+
"tabs_state" : "{\n \"tab_id\": \"#tab1070475139-0\"\n}",
99+
"pivot_chart_state" : "{\n \"hiddenAttributes\": [],\n \"menuLimit\": 200,\n \"cols\": [],\n \"rows\": [],\n \"vals\": [],\n \"exclusions\": {},\n \"inclusions\": {},\n \"unusedAttrsVertical\": 85,\n \"autoSortUnusedAttrs\": false,\n \"inclusionsInfo\": {},\n \"aggregatorName\": \"Count\",\n \"rendererName\": \"Table\"\n}"
100+
},
101+
"id" : "EE80B60DB2C645D58B30EB2B793A5BEC"
102+
},
103+
"cell_type" : "code",
104+
"source" : "display(dataset.toDF)",
105+
"outputs" : [ ]
106+
}, {
107+
"metadata" : {
108+
"trusted" : true,
109+
"input_collapsed" : false,
110+
"collapsed" : false,
111+
"id" : "7797C8DBEB7643D788F4D14F6C8E2B40"
112+
},
113+
"cell_type" : "code",
114+
"source" : "val sum = dataset.map(_._2).reduce(_+_)\n\nprintln(sum)",
115+
"outputs" : [ {
116+
"name" : "stdout",
117+
"output_type" : "stream",
118+
"text" : "501500\nsum: Int = 501500\n"
119+
}, {
120+
"metadata" : { },
121+
"data" : {
122+
"text/html" : ""
123+
},
124+
"output_type" : "execute_result",
125+
"execution_count" : 11,
126+
"time" : "Took: 1.146s, at 2017-05-16 12:20"
127+
} ]
128+
}, {
129+
"metadata" : {
130+
"trusted" : true,
131+
"input_collapsed" : false,
132+
"collapsed" : true,
133+
"id" : "73F60C7C9F2945E38AC0C252F2C3AC1E"
134+
},
135+
"cell_type" : "code",
136+
"source" : "",
137+
"outputs" : [ ]
138+
} ],
139+
"nbformat" : 4
140+
}

‎simple-spark-sources/src/main/resources/notebook.snb

Lines changed: 1 addition & 13 deletions
Large diffs are not rendered by default.

‎simple-spark-sources/src/main/scala/App.scala

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11

2-
package generated.simplespark
2+
package com.example.simplespark
33

44
object Main {
55

@@ -17,12 +17,8 @@ object Main {
1717

1818
// Spark notebook widgets (can be removed if you do not use them)
1919
// Dummy implementation of the most common ones (to avoid shipping 80+ MB of spark-notebook jars)
20-
def display[C](originalData:C, fields:Option[(String, String)]=None, maxPoints:Int=0) = {}
21-
def pairs[C](originalData:C, maxPoints:Int=0) = {}
22-
def ul(capacity:Int=10, initData:Seq[String]=Nil, prefill:Option[String]=None) = {}
23-
def ol(capacity:Int=10, initData:Seq[String]=Nil, prefill:Option[String]=None) = {}
24-
def img(tpe:String="png", width:String="", height:String="") = {}
25-
def text(value: String) = {}
20+
import notebook.front.widgets.charts._
21+
import notebook.front.widgets.charts.MockedCharts._
2622

2723
// Create spark configuration holder
2824
val sparkConf = new SparkConf()
@@ -42,7 +38,7 @@ object Main {
4238
// This is needed only if not using spark-submit (comment otherwise)
4339

4440
def setExecutorJars() = {
45-
val currentProjectJars = Array("lib/generated.simple-spark-0.0.1-SNAPSHOT.jar", "target/scala-2.10/generated.simple-spark_2.10-0.0.1-SNAPSHOT.jar").map{j => new java.io.File(j)}.filter(_.exists()).map(_.getAbsolutePath)
41+
val currentProjectJars = Array("lib/com.example.simple-spark-0.0.1-SNAPSHOT.jar", "target/scala-2.10/com.example.simple-spark_2.10-0.0.1-SNAPSHOT.jar").map{j => new java.io.File(j)}.filter(_.exists()).map(_.getAbsolutePath)
4642
val sparkLibDir = new java.io.File("spark-lib")
4743
val fromProjectJars = Array[String]().map{j => new java.io.File(sparkLibDir, j).getAbsolutePath}
4844
val jarsArray = (sparkConf.get("spark.jars", "").split(",").toArray ++ currentProjectJars ++ fromProjectJars).distinct.filter(!_.isEmpty)

‎simple-spark-sources/src/main/scala/Classes.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11

2-
package generated.simplespark
2+
package com.example.simplespark
33

44

55
//---//

‎simple-spark-sources/src/main/scala/MockedCharts.scala

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
2+
package notebook.front.widgets.charts
3+
4+
import notebook.front.widgets.charts.MockedCharts.DEFAULT_MAX_POINTS
5+
6+
object MockedCharts {
7+
val DEFAULT_MAX_POINTS = 1000
8+
9+
def display[C](originalData:C, fields:Option[(String, String)]=None, maxPoints:Int=0) = {}
10+
def pairs[C](originalData:C, maxPoints:Int=0) = {}
11+
def ul(capacity:Int=10, initData:Seq[String]=Nil, prefill:Option[String]=None) = {}
12+
def ol(capacity:Int=10, initData:Seq[String]=Nil, prefill:Option[String]=None) = {}
13+
def img(tpe:String="png", width:String="", height:String="") = {}
14+
def text(value: String) = {}
15+
}
16+
17+
case class CustomC3Chart[C](
18+
originalData: C,
19+
chartOptions: String = "{}",
20+
sizes: (Int, Int) = (600, 400),
21+
maxPoints: Int = DEFAULT_MAX_POINTS
22+
)
23+
24+
case class ScatterChart[C](
25+
originalData: C,
26+
fields: Option[(String, String)] = None,
27+
sizes: (Int, Int) = (600, 400),
28+
maxPoints: Int = DEFAULT_MAX_POINTS,
29+
groupField: Option[String] = None
30+
)
31+
32+
case class LineChart[C](
33+
originalData: C,
34+
fields: Option[(String, String)] = None,
35+
sizes: (Int, Int) = (600, 400),
36+
maxPoints: Int = DEFAULT_MAX_POINTS,
37+
groupField: Option[String] = None
38+
)
39+
40+
case class RadarChart[C](
41+
originalData: C,
42+
labelField: Option[String] = None,
43+
sizes: (Int, Int) = (600, 400),
44+
maxPoints: Int = DEFAULT_MAX_POINTS
45+
)
46+
47+
48+
case class ParallelCoordChart[C](
49+
originalData: C,
50+
sizes: (Int, Int) = (600, 400),
51+
maxPoints: Int = DEFAULT_MAX_POINTS
52+
)
53+
54+
case class TimeseriesChart[C](
55+
originalData: C,
56+
fields: Option[(String, String)] = None,
57+
sizes: (Int, Int) = (600, 400),
58+
maxPoints: Int = DEFAULT_MAX_POINTS,
59+
groupField: Option[String] = None,
60+
tickFormat: String = "%Y-%m-%d %H:%M:%S"
61+
)
62+
63+
case class BarChart[C](
64+
originalData: C,
65+
fields: Option[(String, String)] = None,
66+
sizes: (Int, Int) = (600, 400),
67+
maxPoints: Int = DEFAULT_MAX_POINTS,
68+
groupField: Option[String] = None
69+
)
70+
71+
72+
case class PieChart[C](
73+
originalData: C,
74+
fields: Option[(String, String)] = None,
75+
sizes: (Int, Int) = (600, 400),
76+
maxPoints: Int = DEFAULT_MAX_POINTS
77+
)
78+
79+
case class DiyChart[C](
80+
originalData: C,
81+
js: String = "",
82+
sizes: (Int, Int) = (600, 400),
83+
maxPoints: Int = DEFAULT_MAX_POINTS
84+
)
85+
86+
87+
case class GeoPointsChart[C](
88+
originalData: C,
89+
sizes: (Int, Int) = (600, 400),
90+
maxPoints: Int = DEFAULT_MAX_POINTS,
91+
latLonFields: Option[(String, String)] = None,
92+
rField: Option[String] = None,
93+
colorField: Option[String] = None
94+
)
95+
96+
97+
case class GeoChart[C](
98+
originalData: C,
99+
sizes: (Int, Int) = (600, 400),
100+
maxPoints: Int = DEFAULT_MAX_POINTS,
101+
geometryField: Option[String] = None,
102+
rField: Option[String] = None,
103+
colorField: Option[String] = None,
104+
fillColorField: Option[String] = None
105+
)
106+
107+
case class GraphChart[C](
108+
originalData: C,
109+
sizes: (Int, Int) = (600, 400),
110+
maxPoints: Int = DEFAULT_MAX_POINTS,
111+
charge: Int = -30,
112+
linkDistance: Int = 20,
113+
linkStrength: Double = 1.0
114+
)
115+
116+
117+
case class PivotChart[C](
118+
originalData: C,
119+
sizes: (Int, Int) = (600, 400),
120+
maxPoints: Int = DEFAULT_MAX_POINTS,
121+
// FIXME: otherwise this would add dependency on play-json!
122+
// derivedAttributes:JsObject=play.api.libs.json.Json.obj(),
123+
options: Map[String, String] = Map.empty
124+
)
125+
126+
127+
case class CustomPlotlyChart[C](
128+
originalData: C,
129+
layout: String = "{}",
130+
dataOptions: String = "{}",
131+
dataSources: String = "{}",
132+
sizes: (Int, Int) = (600, 400),
133+
maxPoints: Int = DEFAULT_MAX_POINTS
134+
)
135+
136+
case class TableChart[C](
137+
originalData: C,
138+
filterCol: Option[Seq[String]] = None,
139+
sizes: (Int, Int) = (600, 400),
140+
maxPoints: Int = DEFAULT_MAX_POINTS
141+
)
142+
143+

0 commit comments

Comments
 (0)