Scala 我们可以通过Spark流使用来自主题的JMS消息吗？

问题：有一个JMS主题，我有以下连接详细信息： 网址：xyz 连接工厂：jms.xyz 主题名称：jms.xyz 用户名： 密码：

是否有工作代码在Spark Scala中创建订阅服务器，该订阅服务器使用来自主题的JMS消息

我尝试过使用spark streaming的socketTextStream函数，但它只有URL参数。 我正在寻找一个spark流媒体功能，它可以包含我所有的5个参数： 1） 网址 2） 连接工厂 3） 主题名称 4） 用户名 5） 密码

我试过在Spark Shell中跑步

我正在寻找一个spark streaming函数，它可以包含我所有的5个参数和一个可以使用本主题JMS消息的spark scala代码： 1） 网址 2） 连接工厂 3） 主题名称 4） 用户名 5） 密码

我正在寻找Spark Shell基本命令，我可以逐行执行

Q：我们可以通过Spark流使用来自某个主题的JMS消息吗

对。恐怕没有解决这个问题的灵丹妙药

根据消息传递提供程序的不同，实现可能会有所不同。 你可能需要为此写一封信

参见示例2，该示例使用jms主题与spark流媒体集成

示例1（）：

示例2使用and:

示例3:Solace使用了spark的定制接收器：这种方式我很久以前就在spark 1.3出现时工作过

---

进一步阅读：

您正在使用哪个jms提供程序？谢谢您的评论。解决方案是针对队列接收器的，但我希望在Spark Streaming中开发一个主题订阅者

import org.apache.log4j.Logger;
import org.apache.spark.storage.StorageLevel;
import org.apache.spark.streaming.receiver.Receiver;

import javax.jms.*;
import javax.naming.Context;
import java.util.Hashtable;

public class JMSReceiver extends Receiver<JMSEvent> implements MessageListener
{
    private static final Logger log = Logger.getLogger(JMSReceiver.class);

    private static final String JNDI_INITIAL_CONTEXT_FACTORY       = "org.apache.qpid.jms.jndi.JmsInitialContextFactory";
    private static final String JNDI_CONNECTION_FACTORY_NAME       = "JMSReceiverConnectionFactory";
    private static final String JNDI_QUEUE_NAME                    = "JMSReceiverQueue";
    private static final String JNDI_CONNECTION_FACTORY_KEY_PREFIX = "connectionfactory.";
    private static final String JNDI_QUEUE_KEY_PREFIX              = "queue.";

    private StorageLevel _storageLevel;

    private String _brokerURL;
    private String _username;
    private String _password;
    private String _queueName;
    private String _selector;

    private Connection _connection;

    public JMSReceiver(String brokerURL, String username, String password, String queueName, String selector, StorageLevel storageLevel)
    {
        super(storageLevel);
        _storageLevel = storageLevel;
        _brokerURL = brokerURL;
        _username = username;
        _password = password;
        _queueName = queueName;
        _selector = selector;

        log.info("Constructed" + this);
    }

    @Override
    public void onMessage(Message message)
    {
        try
        {
            log.info("Received: " + message);
            JMSEvent jmsEvent = new JMSEvent(message);
            store(jmsEvent);
        } catch (Exception exp)
        {
            log.error("Caught exception converting JMS message to JMSEvent", exp);
        }
    }

    @Override
    public StorageLevel storageLevel()
    {
        return _storageLevel;
    }

    public void onStart()
    {

        log.info("Starting up...");

        try
        {

            Hashtable<Object, Object> env = new Hashtable<Object, Object>();
            env.put(Context.INITIAL_CONTEXT_FACTORY, JNDI_INITIAL_CONTEXT_FACTORY);
            env.put(JNDI_CONNECTION_FACTORY_KEY_PREFIX + JNDI_CONNECTION_FACTORY_NAME, _brokerURL);
            env.put(JNDI_QUEUE_KEY_PREFIX + JNDI_QUEUE_NAME, _queueName);
            javax.naming.Context context = new javax.naming.InitialContext(env);

            ConnectionFactory factory = (ConnectionFactory) context.lookup(JNDI_CONNECTION_FACTORY_NAME);
            Destination queue = (Destination) context.lookup(JNDI_QUEUE_NAME);

            if ((_username == null) || (_password == null))
            {
                _connection = factory.createConnection();
            } else
            {
                _connection = factory.createConnection(_username, _password);
            }
            _connection.setExceptionListener(new JMSReceiverExceptionListener());

            Session session = _connection.createSession(false, Session.AUTO_ACKNOWLEDGE);

            MessageConsumer messageConsumer;

            if (_selector != null)
            {
                messageConsumer = session.createConsumer(queue, _selector);
            } else
            {
                messageConsumer = session.createConsumer(queue);
            }
            messageConsumer.setMessageListener(this);

            _connection.start();

            log.info("Completed startup.");
        } catch (Exception exp)
        {
            // Caught exception, try a restart
            log.error("Caught exception in startup", exp);
            restart("Caught exception, restarting.", exp);
        }
    }

    public void onStop()
    {
        // Cleanup stuff (stop threads, close sockets, etc.) to stop receiving data

        log.info("Stopping...");
        try
        {
            _connection.close();
        } catch (JMSException exp)
        {
            log.error("Caught exception stopping", exp);
        }
        log.info("Stopped.");
    }

    private class JMSReceiverExceptionListener implements ExceptionListener
    {
        @Override
        public void onException(JMSException exp)
        {
            log.error("Connection ExceptionListener fired, attempting restart.", exp);
            restart("Connection ExceptionListener fired, attempting restart.");
        }
    }

    @Override
    public String toString()
    {
        return "JMSReceiver{" +
                "brokerURL='" + _brokerURL + '\'' +
                ", username='" + _username + '\'' +
                ", password='" + _password + '\'' +
                ", queueName='" + _queueName + '\'' +
                ", selector='" + _selector + '\'' +
                '}';
    }
}

import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming._
import org.apache.spark.streaming.dstream._
import org.apache.spark.streaming.receiver.Receiver

private[streaming]
class JMSInputDStream(
                       @transient ssc_ : StreamingContext,
                       brokerURL: String,
                       username: String,
                       password: String,
                       queuename: String,
                       selector: String,
                       storageLevel: StorageLevel
                       ) extends ReceiverInputDStream[JMSEvent](ssc_) {

  override def getReceiver(): Receiver[JMSEvent] = {
    new JMSReceiver(brokerURL, username, password, queuename, selector, storageLevel)
  }
}

import org.apache.spark.Logging
import org.apache.spark.storage.StorageLevel
import javax.{jms => jms}

/** Simple class of a receiver that can be run on worker nodes to receive the data from JMS Topic.
  *
  * In JMS a Topic implements publish and subscribe semantics.
  * When you publish a message it goes to all the subscribers who are interested - so zero to many subscribers will receive a copy of the message.
  * Only subscribers who had an active subscription at the time the broker receives the message will get a copy of the message.
  *
  * {{{
  *  val sc: SparkContext = SparkContext.getOrCreate(conf)
  *  val ssc: StreamingContext = new StreamingContext(sc, Seconds(...))
  *
  *  val stream: InputDStream[String] = ssc.receiverStream(new JmsTopicReceiver(
  *    topicName = "testTopic",
  *    transformer = { msg => msg.asInstanceOf[javax.jms.TextMessage].getText() },
  *    connectionProvider = { () => {
  *      val cf = new org.apache.activemq.ActiveMQConnectionFactory("tcp://localhost:61616")
  *      cf.setOptimizeAcknowledge(true)
  *      cf.createConnection("username", "password")
  *    }}
  *  ))
  *
  *  ...
  *
  *  ssc.start()
  *  ssc.awaitTermination()
  * }}}
  *
  * @param connectionProvider provides <CODE>javax.jms.Connection</CODE> for the receiver.
  * @param transformer (pre)transforms <CODE>javax.jms.Message</CODE> to appropriate class (it's required to do this before populate the result).
  * @param topicName the name of required <CODE>javax.jms.Topic</CODE>.
  * @param messageSelector only messages with properties matching the message selector expression are delivered.
  * @param storageLevel flags for controlling the storage of an RDD.
  * @tparam T RDD element type.
  */
class JmsTopicReceiver[T] (
  connectionProvider: (() => jms.Connection),
  transformer: (jms.Message => T),
  topicName: String,
  messageSelector: Option[String] = None,
  storageLevel: StorageLevel = StorageLevel.MEMORY_AND_DISK_SER_2
) extends AbstractJmsReceiver[T](
  messageSelector = messageSelector,
  storageLevel = storageLevel
) with Logging {

  override protected def buildConnection(): jms.Connection = connectionProvider()
  override protected def transform(message: jms.Message): T = transformer(message)
  override protected def buildDestination(session: jms.Session): jms.Destination = session.createTopic(topicName)

}