Sql server 如何使用Kafka获取SQL表?
我想用Kafka(Confluent 5.4.0和Kafka 2.12)在SQL中查看查询结果 我首先尝试使用Kafka JDBC连接器(使用mssql-JDBC-8.2.0.jre8驱动程序和Java 8u201)和以下参数:Sql server 如何使用Kafka获取SQL表?,sql-server,jdbc,apache-kafka,Sql Server,Jdbc,Apache Kafka,我想用Kafka(Confluent 5.4.0和Kafka 2.12)在SQL中查看查询结果 我首先尝试使用Kafka JDBC连接器(使用mssql-JDBC-8.2.0.jre8驱动程序和Java 8u201)和以下参数: { "name": "sqlview", "connector.class": "io.confluent.connect.jdbc.JdbcSourceConnector", "tasks.max": "10", "value.converter":
{
"name": "sqlview",
"connector.class": "io.confluent.connect.jdbc.JdbcSourceConnector",
"tasks.max": "10",
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
"connection.url": "jdbc:sqlserver://localhost:1433;DatabaseName=testDB",
"connection.user": "testUser",
"connection.password": "password",
"mode": "timestamp+incrementing",
"incrementing.column.name": "ID",
"timestamp.column.name": "CREACIONDATE",
"query": "select ROW_NUMBER() OVER(ORDER BY t1.CDATE ASC) AS ID, t1.CDATE as CREACIONDATE, t1.NAME, t2.CRITICALITY, t3.DESCRIPTION, t4.USER, COALESCE(t5.name, 'None') as task from WORKSPACE w inner join TABLE1 t1 on w.IDWORKSPACE = t1.IDWORKSPACE inner join TABLE2 t2 on t1.IDASSET = t2.IDASSET inner join TABLE3 t3 on t2.IDASSET = t3.IDASSET inner join TABLE4 t4 on t2.IDUSER = t4.IDUSER inner join TABLE5 t5 on t2.IDASSET = t5.IDASSET",
"topic.prefix": "conceptTest"
}
{
"name": "sqlview",
"connector.class": "io.confluent.connect.jdbc.JdbcSourceConnector",
"tasks.max": "10",
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
"connection.url": "jdbc:sqlserver://localhost:1433;DatabaseName=testDB",
"connection.user": "testUser",
"connection.password": "password",
"mode": "incrementing",
"query": "select ROW_NUMBER() OVER(ORDER BY t1.CDATE ASC) AS ID, t1.CDATE as CREACIONDATE, t1.NAME, t2.CRITICALITY, t3.DESCRIPTION, t4.USER, COALESCE(t5.name, 'None') as task from WORKSPACE w inner join TABLE1 t1 on w.IDWORKSPACE = t1.IDWORKSPACE inner join TABLE2 t2 on t1.IDASSET = t2.IDASSET inner join TABLE3 t3 on t2.IDASSET = t3.IDASSET inner join TABLE4 t4 on t2.IDUSER = t4.IDUSER inner join TABLE5 t5 on t2.IDASSET = t5.IDASSET",
"topic.prefix": "conceptTest"
}
但是卡夫卡无法创建主题,卡夫卡连接日志会显示下一条错误消息:
错误:无法运行表的查询
timestampincrementablequeryer{table=null,query='select
(按t1.CDATE ASC排序)上的行号()作为ID,t1.CDATE作为
CREACIONDATE,t1.名称,t2.临界性,t3.描述,t4.用户,
将(t5.name,'None')合并为工作区中的任务w内部联接表1
w.IDWORKSPACE上的t1=t1.IDWORKSPACE内部联接表2上的t2
t1.IDASSET=t2.IDASSET内部连接表3 t2.IDASSET上的t3=
t3.IDASSET内部联接表4 t2上的t4.IDUSER=t4.IDUSER内部联接
表5 t2.IDASSET=t5.IDASSET',topicPrefix='conceptTest',上的t5,
incrementingColumn='ID',timestampColumns=[CREACIONDATE]}:{}
(io.confluent.connect.jdbc.source.JdbcSourceTask:337)
com.microsoft.sqlserver.jdbc.SQLServerException:列名无效
“CREACIONDATE”。在
com.microsoft.sqlserver.jdbc.SQLServerException.makeFromDatabaseError(SQLServerException.java:262)
在
com.microsoft.sqlserver.jdbc.SQLServerStatement.getNextResult(SQLServerStatement.java:1624)
在
com.microsoft.sqlserver.jdbc.SQLServerPreparedStatement.doExecutePreparedStatement(SQLServerPreparedStatement.java:594)
在
com.microsoft.sqlserver.jdbc.SQLServerPreparedStatement$PrepStmtExecCmd.doExecute(SQLServerPreparedStatement.java:524)
在
com.microsoft.sqlserver.jdbc.TDSCommand.execute(IOBuffer.java:7194)
在
com.microsoft.sqlserver.jdbc.SQLServerConnection.executeCommand(SQLServerConnection.java:2979)
在
com.microsoft.sqlserver.jdbc.SQLServerStatement.executeCommand(SQLServerStatement.java:248)
在
com.microsoft.sqlserver.jdbc.SQLServerStatement.executeStatement(SQLServerStatement.java:223)
在
com.microsoft.sqlserver.jdbc.SQLServerPreparedStatement.executeQuery(SQLServerPreparedStatement.java:446)
在
io.confluent.connect.jdbc.source.TimestampIncrementingTableQuerier.executeQuery(TimestampIncrementingTableQuerier.java:178)
在
io.confluent.connect.jdbc.source.TableQuerier.maybeStartQuery(TableQuerier.java:92)
在
io.confluent.connect.jdbc.source.TimestampIncrementingTableQuerier.maybeStartQuery(TimestampIncrementingTableQuerier.java:61)
在
io.confluent.connect.jdbc.source.JdbcSourceTask.poll(JdbcSourceTask.java:315)
在
org.apache.kafka.connect.runtime.WorkerSourceTask.poll(WorkerSourceTask.java:265)
在
org.apache.kafka.connect.runtime.WorkerSourceTask.execute(WorkerSourceTask.java:232)
在
org.apache.kafka.connect.runtime.WorkerTask.doRun(WorkerTask.java:177)
在
org.apache.kafka.connect.runtime.WorkerTask.run(WorkerTask.java:227)
在
Executors$RunnableAdapter.call(Executors.java:511)
在java.util.concurrent.FutureTask.run(FutureTask.java:266)处
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
在
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
运行(Thread.java:748)
然后更改此项的参数:
{
"name": "sqlview",
"connector.class": "io.confluent.connect.jdbc.JdbcSourceConnector",
"tasks.max": "10",
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
"connection.url": "jdbc:sqlserver://localhost:1433;DatabaseName=testDB",
"connection.user": "testUser",
"connection.password": "password",
"mode": "incrementing",
"incrementing.column.name": "ID",
"query": "select ROW_NUMBER() OVER(ORDER BY t1.CDATE ASC) AS ID, t1.CDATE as CREACIONDATE, t1.NAME, t2.CRITICALITY, t3.DESCRIPTION, t4.USER, COALESCE(t5.name, 'None') as task from WORKSPACE w inner join TABLE1 t1 on w.IDWORKSPACE = t1.IDWORKSPACE inner join TABLE2 t2 on t1.IDASSET = t2.IDASSET inner join TABLE3 t3 on t2.IDASSET = t3.IDASSET inner join TABLE4 t4 on t2.IDUSER = t4.IDUSER inner join TABLE5 t5 on t2.IDASSET = t5.IDASSET",
"topic.prefix": "conceptTest"
}
同样,卡夫卡无法创建主题,日志会显示下一条消息:
错误:无法运行表的查询
timestampincrementablequeryer{table=null,query='select
(按t1.CDATE ASC排序)上的行号()作为ID,t1.CDATE作为
CREACIONDATE,t1.名称,t2.临界性,t3.描述,t4.用户,
将(t5.name,'None')合并为工作区中的任务w内部联接表1
w.IDWORKSPACE上的t1=t1.IDWORKSPACE内部联接表2上的t2
t1.IDASSET=t2.IDASSET内部连接表3 t2.IDASSET上的t3=
t3.IDASSET内部联接表4 t2上的t4.IDUSER=t4.IDUSER内部联接
表5 t2.IDASSET=t5.IDASSET',topicPrefix='conceptTest',上的t5,
incrementingColumn='ID',timestampColumns=[]}:{}
(io.confluent.connect.jdbc.source.JdbcSourceTask:337)
com.microsoft.sqlserver.jdbc.SQLServerException:列名无效
“ID”。在
com.microsoft.sqlserver.jdbc.SQLServerException.makeFromDatabaseError(SQLServerException.java:262)
在
com.microsoft.sqlserver.jdbc.SQLServerStatement.getNextResult(SQLServerStatement.java:1624)
在
com.microsoft.sqlserver.jdbc.SQLServerPreparedStatement.doExecutePreparedStatement(SQLServerPreparedStatement.java:594)
在
com.microsoft.sqlserver.jdbc.SQLServerPreparedStatement$PrepStmtExecCmd.doExecute(SQLServerPreparedStatement.java:524)
在
com.microsoft.sqlserver.jdbc.TDSCommand.execute(IOBuffer.java:7194)
在
com.microsoft.sqlserver.jdbc.SQLServerConnection.executeCommand(SQLServerConnection.java:2979)
在
com.microsoft.sqlserver.jdbc.SQLServerStatement.executeCommand(SQLServerStatement.java:248)
在
com.microsoft.sqlserver.jdbc.SQLServerStatement.executeStatement(SQLServerStatement.java:223)
在
com.microsoft.sqlserver.jdbc.SQLServerPreparedStatement.executeQuery(SQLServerPreparedStatement.java:446)
在
io.confluent.connect.jdbc.source.TimestampIncrementingTableQuerier.executeQuery(TimestampIncrementingTableQuerier.java:178)
在
io.confluent.connect.jdbc.source.TableQuerier.maybeStartQuery(TableQuerier.java:92)
在
io.confluent.connect.jdbc.source.TimestampIncrementingTableQuerier.maybeStartQuery(TimestampIncrementingTableQuerier.java:61)
在
io.confluent.connect.jdbc.source.JdbcSourceTask.poll(JdbcSourceTask.java:315)
在
org.apache.kafka.connect.runtime.WorkerSourceTask.poll(WorkerSourceTask.java:265)
在
org.apache.kafka.connect.runtime.WorkerSourceTask.execute(WorkerSourceTask.java:232)
在
org.apache.kafka.connect.runtime.WorkerTask.doRun(WorkerTask.java:177)
在
org.apache.kafka.connect.runtime.WorkerTask.run(WorkerTask.java:227)
在
Executors$RunnableAdapter.call(Executors.java:511)
在java.util.concurrent