Mysql 粗粒化错误火花
我使用Spark 2.1.1和Scala 2.10，Spark单机版有两个节点 我想执行一个Spark代码，从MySQL表中读取数据集并将其写入Cassandra表 代码：Mysql 粗粒化错误火花,mysql,scala,apache-spark,cassandra,Mysql,Scala,Apache Spark,Cassandra,我使用Spark 2.1.1和Scala 2.10，Spark单机版有两个节点 我想执行一个Spark代码，从MySQL表中读取数据集并将其写入Cassandra表 代码： object RdmsToCassandra extends Serializable { def main(args: Array[String]) { val conf = new SparkConf().set("spark.cassandra.connection.host", "192.168.22.231"
object RdmsToCassandra extends Serializable {
def main(args: Array[String]) {
val conf = new SparkConf().set("spark.cassandra.connection.host", "192.168.22.231")
val sc = new SparkContext("spark://192.168.22.231:7077", "MigrateMySQLToCassandra", conf)
val mysqlJdbcString: String = s"jdbc:mysql://192.168.22.238/customer_events?user=root&password=qweqwe"
Class.forName("com.mysql.jdbc.Driver").newInstance
CassandraConnector(conf).withSessionDo { session =>
session.execute("CREATE KEYSPACE IF NOT EXISTS test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 2 }")
session.execute("CREATE TABLE IF NOT EXISTS test.store( store_name text PRIMARY KEY, location text, store_type text)" )
}
val highestId: Long = 2
val startingId: Long = 0
val numberOfPartitions = 1;
val customerEvents = new JdbcRDD(sc, () => { DriverManager.getConnection(mysqlJdbcString)},
"select * from store limit ?, ?",startingId, highestId, numberOfPartitions,
(r: ResultSet) => {
(r.getString("store_name"),
r.getString("location"),
r.getString("store_type")
)
}
)
customerEvents.saveToCassandra("test", "store1",
SomeColumns("store_name"))
}
}
spark-submit --master spark://192.168.22.231:6066 \
--class "RdmsToCassandra" \
rdbmstocassandra_2.10-1.0.jar
有人能告诉我这里有什么问题吗。我知道这有点老了，但可能对路过的其他人有帮助： 我遇到了一个类似的问题，通过对NodeManager日志的调查，我找到了根本原因。 您可以通过Web界面访问它们
nodeManagerAdress:PORT/logs
object RdmsToCassandra extends Serializable {
def main(args: Array[String]) {
val conf = new SparkConf().set("spark.cassandra.connection.host", "192.168.22.231")
val sc = new SparkContext("spark://192.168.22.231:7077", "MigrateMySQLToCassandra", conf)
val mysqlJdbcString: String = s"jdbc:mysql://192.168.22.238/customer_events?user=root&password=qweqwe"
Class.forName("com.mysql.jdbc.Driver").newInstance
CassandraConnector(conf).withSessionDo { session =>
session.execute("CREATE KEYSPACE IF NOT EXISTS test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 2 }")
session.execute("CREATE TABLE IF NOT EXISTS test.store( store_name text PRIMARY KEY, location text, store_type text)" )
}
val highestId: Long = 2
val startingId: Long = 0
val numberOfPartitions = 1;
val customerEvents = new JdbcRDD(sc, () => { DriverManager.getConnection(mysqlJdbcString)},
"select * from store limit ?, ?",startingId, highestId, numberOfPartitions,
(r: ResultSet) => {
(r.getString("store_name"),
r.getString("location"),
r.getString("store_type")
)
}
)
customerEvents.saveToCassandra("test", "store1",
SomeColumns("store_name"))
}
}
nodeManagerAdress:PORT/conf
我知道它有点老了，但可能对路过的其他人有帮助： 我遇到了一个类似的问题，通过对NodeManager日志的调查，我找到了根本原因。 您可以通过Web界面访问它们
nodeManagerAdress:PORT/logs
object RdmsToCassandra extends Serializable {
def main(args: Array[String]) {
val conf = new SparkConf().set("spark.cassandra.connection.host", "192.168.22.231")
val sc = new SparkContext("spark://192.168.22.231:7077", "MigrateMySQLToCassandra", conf)
val mysqlJdbcString: String = s"jdbc:mysql://192.168.22.238/customer_events?user=root&password=qweqwe"
Class.forName("com.mysql.jdbc.Driver").newInstance
CassandraConnector(conf).withSessionDo { session =>
session.execute("CREATE KEYSPACE IF NOT EXISTS test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 2 }")
session.execute("CREATE TABLE IF NOT EXISTS test.store( store_name text PRIMARY KEY, location text, store_type text)" )
}
val highestId: Long = 2
val startingId: Long = 0
val numberOfPartitions = 1;
val customerEvents = new JdbcRDD(sc, () => { DriverManager.getConnection(mysqlJdbcString)},
"select * from store limit ?, ?",startingId, highestId, numberOfPartitions,
(r: ResultSet) => {
(r.getString("store_name"),
r.getString("location"),
r.getString("store_type")
)
}
)
customerEvents.saveToCassandra("test", "store1",
SomeColumns("store_name"))
}
}
nodeManagerAdress:PORT/conf