Fatal编程技术网
  • apache-flink/
  • Apache flink 弗林克什么时候到期；来自Queryablestate的s时间窗口结果？

Apache flink 弗林克什么时候到期；来自Queryablestate的s时间窗口结果？

apache-flink

Apache flink 弗林克什么时候到期；来自Queryablestate的s时间窗口结果？,apache-flink,Apache Flink,我已经用翻滚窗口和QueryableState实现了Total WordCount示例 我使用了10秒时间窗口，当我打印结果时，它会显示正确的结果，但当我使用queryable状态并使用QueryableClient进行查询时，它会缓存时间窗口的最后一个结果，即使时间窗口发生更改 e、 g，对于时间窗口11:00:01到11:00:10，“Nirav”的字数为5 当我在时间11:00:50上查询“Nirav”时，它返回之前的计数5 所以我有两个问题： 这是Flink的QueryableState

我已经用翻滚窗口和QueryableState实现了Total WordCount示例

我使用了10秒时间窗口，当我打印结果时，它会显示正确的结果，但当我使用queryable状态并使用QueryableClient进行查询时，它会缓存时间窗口的最后一个结果，即使时间窗口发生更改

e、 g，对于时间窗口11:00:01到11:00:10，“Nirav”的字数为5

当我在时间11:00:50上查询“Nirav”时，它返回之前的计数5

所以我有两个问题：

  • 这是Flink的QueryableStateClient的默认行为吗？它缓存同一密钥的最后一个输出，直到该密钥的新状态
  • 时间窗口结束时，如何清除以前的结果 下面是可查询的实现

    int sec = 10;
        Time seconds = Time.seconds(sec);
        text.flatMap(new FlatMapFunction<String, WordWithCount>() {
            public void flatMap(String value, Collector<WordWithCount> out) {
                for (String word : value.split("\\s")) {
                    out.collect(new WordWithCount(word, 1L));
                }
            }
        })
        .keyBy("word")
        .timeWindow(seconds)
        .reduce(new ReduceFunction<WordWithCount>() {
            public WordWithCount reduce(WordWithCount a, WordWithCount b) {
                System.out.println("After time window fun:- a.word:" + a.word + ", a.count:" + a.count + ", b.word:" + b.word + ", b.count:" + b.count);
                return new WordWithCount(a.word, a.count + b.count);
            }
        })
        .keyBy(wordWithCount -> wordWithCount.word)
        .asQueryableState("wordCountQuery", valueStateDescriptor)

    int-sec=10；
时间秒=时间秒（秒）；
text.flatMap（新的flatMap函数（）{
公共void平面图（字符串值，收集器输出）{
for（字符串字：value.split（\\s）））{
out.collect（新单词with count（单词，1L））；
}
}
})
.keyBy（“word”）
.时间窗口（秒）
.reduce（新的ReduceFunction（）{
公共单词WithCount reduce（单词WithCount a、单词WithCount b）{
System.out.println（“时间窗口后的乐趣：-a.word:+a.word+”，a.count:+a.count+，b.word:+b.word+，b.count:+b.count”）；
使用count返回新词（a.word，a.count+b.count）；
}
})
.keyBy（wordWithCount->wordWithCount.word）
.asQueryableState（“wordCountQuery”，valueStateDescriptor）
    整体实施

    socketwindwindoworcountwithqueryablestatewithtimewindow.java 包com.nirav.modi

    import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.state.ReducingStateDescriptor;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.streaming.api.CheckpointingMode;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.util.Collector;

public class SocketWindowWordCountWithQueryableStateWithTimeWindow {

    public static void main(String[] args) throws Exception {

        // the port to connect to
        final int port;
        try {
            final ParameterTool params = ParameterTool.fromArgs(args);
            port = params.getInt("port");
        } catch (Exception e) {
            System.err.println("No port specified. Please run 'SocketWindowWordCount --port <port>'");
            return;
        }

        // get the execution environment
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.enableCheckpointing(10000, CheckpointingMode.EXACTLY_ONCE);

        // get input data by connecting to the socket
        DataStream<String> text = env.socketTextStream("localhost", port);

        ReduceFunction<WordWithCount> reduceFunction = new ReduceFunction<WordWithCount>() {
            public WordWithCount reduce(WordWithCount a, WordWithCount b) {
                System.out.println("reduce fun:- a.word:" + a.word + ", a.count:" + a.count + ", b.word:" + b.word + ", b.count:" + b.count);
                return new WordWithCount(a.word, a.count + b.count);
            }
        };
//        ReducingStateDescriptor<WordWithCount> descriptor = new ReducingStateDescriptor<WordWithCount>("wordCountQuery", reduceFunction, WordWithCount.class);

        ValueStateDescriptor<WordWithCount> valueStateDescriptor = new ValueStateDescriptor<WordWithCount>("wordCountQuery", WordWithCount.class);

        int sec = 10;
        Time seconds = Time.seconds(sec);
        text.flatMap(new FlatMapFunction<String, WordWithCount>() {
            public void flatMap(String value, Collector<WordWithCount> out) {
                for (String word : value.split("\\s")) {
                    out.collect(new WordWithCount(word, 1L));
                }
            }
        })
                .keyBy("word")
                .timeWindow(seconds)
                .reduce(new ReduceFunction<WordWithCount>() {
                    public WordWithCount reduce(WordWithCount a, WordWithCount b) {
                        System.out.println("After time window fun:- a.word:" + a.word + ", a.count:" + a.count + ", b.word:" + b.word + ", b.count:" + b.count);
                        return new WordWithCount(a.word, a.count + b.count);
                    }
                }).keyBy(wordWithCount -> wordWithCount.word)
                .asQueryableState("wordCountQuery", valueStateDescriptor);

        env.getConfig().enableSysoutLogging();
        JobGraph jobGraph = env.getStreamGraph().getJobGraph();

        System.out.println("[info] Window WordCount with Time Window Job ID: " + jobGraph.getJobID());
        System.out.println();
        env.execute("Socket Window WordCount with Time Window of " + sec + " seconds");
    }

    // Data type for words with count
    public static class WordWithCount {

        public String word;
        public long count;

        public WordWithCount() {
        }

        public WordWithCount(String word, long count) {
            this.word = word;
            this.count = count;
        }

        @Override
        public String toString() {
            return word + " : " + count;
        }
    }
}

    import org.apache.flink.api.common.functions.FlatMapFunction；
导入org.apache.flink.api.common.functions.ReduceFunction；
导入org.apache.flink.api.common.state.ReduceingStateDescriptor；
导入org.apache.flink.api.common.state.ValueStateDescriptor；
导入org.apache.flink.api.java.utils.ParameterTool；
导入org.apache.flink.runtime.jobgraph.jobgraph；
导入org.apache.flink.streaming.api.CheckpointingMode；
导入org.apache.flink.streaming.api.datastream.datastream；
导入org.apache.flink.streaming.api.environment.StreamExecutionEnvironment；
导入org.apache.flink.streaming.api.windowing.time.time；
导入org.apache.flink.util.Collector；
公共类SocketWindows或CountWithQueryableStateWithTimeWindow{
公共静态void main（字符串[]args）引发异常{
//要连接到的端口
最终国际端口；
试一试{
final ParameterTool params=ParameterTool.fromArgs（args）；
端口=params.getInt（“端口”）；
}捕获（例外e）{
System.err.println（“未指定端口。请运行'SocketWindowWordCount--port'”；
返回；
}
//获取执行环境
最终StreamExecutionEnvironment env=StreamExecutionEnvironment.getExecutionEnvironment（）；
环境启用检查点（10000，检查点模式。仅启用一次）；
//通过连接到套接字获取输入数据
DataStream text=env.socketTextStream（“localhost”，端口）；
ReduceFunction ReduceFunction=新的ReduceFunction（）{
公共单词WithCount reduce（单词WithCount a、单词WithCount b）{
System.out.println（“减少乐趣：-a.word:+a.word+”，a.count:+a.count+”，b.word:+b.word+”，b.count:+b.count）；
使用count返回新词（a.word，a.count+b.count）；
}
};
//ReductionStateDescriptor描述符=新的ReductionStateDescriptor（“wordCountQuery”，ReductionFunction，WordWithCount.class）；
ValueStateDescriptor ValueStateDescriptor=新的ValueStateDescriptor（“wordCountQuery”，WordWithCount.class）；
整数秒=10；
时间秒=时间秒（秒）；
text.flatMap（新的flatMap函数（）{
公共void平面图（字符串值，收集器输出）{
for（字符串字：value.split（\\s）））{
out.collect（新单词with count（单词，1L））；
}
}
})
.keyBy（“word”）
.时间窗口（秒）
.reduce（新的ReduceFunction（）{
公共单词WithCount reduce（单词WithCount a、单词WithCount b）{
System.out.println（“时间窗口后的乐趣：-a.word:+a.word+”，a.count:+a.count+，b.word:+b.word+，b.count:+b.count”）；
使用count返回新词（a.word，a.count+b.count）；
}
}).keyBy（wordWithCount->wordWithCount.word）
.asQueryableState（“wordCountQuery”，valueStateDescriptor）；
env.getConfig（）.enableSysoutLogging（）；
JobGraph JobGraph=env.getStreamGraph（）.getJobGraph（）；
System.out.println（“[info]窗口字数，时间窗口作业ID:”+jobGraph.getJobID（））；
System.out.println（）；
环境执行（“套接字窗口字计数，时间窗口为“+秒+”秒”）；
}
//带计数的字的数据类型
公共静态类WordWithCount{
公共字符串；
公众长时间计数；
PublicWordWithCount（）{
}
公用字WithCount（字符串字、长计数）{
这个单词=单词；
this.count=计数；
}
@凌驾
公共字符串toString（）{
返回单词+“：”+计数；
}
}
}
    QueryStateWithWindowTest.java

    package com.nirav.modi;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.state.ValueState;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.queryablestate.client.QueryableStateClient;
import scala.tools.jline_embedded.console.ConsoleReader;
import java.io.PrintWriter;
import java.net.UnknownHostException;
import java.util.concurrent.CompletableFuture;
public class QueryStateWithWindowTest {
    public static void main(String[] args) throws Exception {
        // the jobId to connect to
        final String jobId;
        final String queryableStateName;
        try {
            final ParameterTool params = ParameterTool.fromArgs(args);
            jobId = params.get("jobId");
            queryableStateName = params.get("queryableStateName");
        } catch (Exception e) {
            System.err.println("No jobId specified. Please run 'SocketWindowWordCount --jobId <jobId>'");
            return;
        }
        try {
            ValueStateDescriptor<SocketWindowWordCountWithQueryableStateWithTimeWindow.WordWithCount> valueStateDescriptor = new ValueStateDescriptor<SocketWindowWordCountWithQueryableStateWithTimeWindow.WordWithCount>("wordCountQuery", SocketWindowWordCountWithQueryableStateWithTimeWindow.WordWithCount.class);
            QueryableStateClient client = new QueryableStateClient("truecomtelesoft", 9069);
            ExecutionConfig config = new ExecutionConfig();
            client.setExecutionConfig(config.enableClosureCleaner());
            ConsoleReader reader = new ConsoleReader();
            reader.setPrompt("$ ");
            PrintWriter out = new PrintWriter(reader.getOutput());
            String line;
            while ((line = reader.readLine()) != null) {
                String key = line.toLowerCase().trim();
                out.printf("[info] Querying key '%s'\n", key);
                try {
                    long start = System.currentTimeMillis();
                    CompletableFuture<ValueState<SocketWindowWordCountWithQueryableStateWithTimeWindow.WordWithCount>> kvState = client.getKvState(JobID.fromHexString(jobId), queryableStateName, key, BasicTypeInfo.STRING_TYPE_INFO, valueStateDescriptor);
                    try {
                        SocketWindowWordCountWithQueryableStateWithTimeWindow.WordWithCount wordWithCount = kvState.get().value();
                        long end = System.currentTimeMillis();
                        long duration = Math.max(0, end - start);
                        out.printf("%d (query took %d ms)\n", wordWithCount.count, duration);
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                } catch (Exception e) {
                    out.println("Query failed because of the following Exception:");
                    e.printStackTrace(out);
                }
            }
        } catch (UnknownHostException e) {
            e.printStackTrace();
        }
    }
}

    package com.nirav.modi；
导入org.apache.flink.api.common.ExecutionConfig；
导入org.apache.flink.api.common.JobID；
导入org.apache.flink.api.common.state.ValueState；
导入org.apache.flink.api.common.state.Valu