Google cloud dataflow 如何在具有小束的流式管道中按N个元素进行批处理？

我已经实现了N个元素的批处理，如以下回答所述：

但当我以流模式运行管道时，有很多批处理只有1或2个元素。据我所知，这是因为包裹太小了。运行一天后，批处理中的元素平均数量大约为4个。我真的需要它接近10，以便更好地执行接下来的步骤

有没有办法控制捆绑包的大小？ 或者我应该为此使用“GroupIntoBatches”转换。在这种情况下，我不清楚应该选择什么作为键

更新：

---

使用java线程id或VM主机名作为密钥来应用“GroupIntoBatches”转换是一个好主意吗？

我最后用“GroupIntoBatches”进行了复合转换。 以下答案包含有关关键点选择的建议：

在我当前的实现中，我使用随机键来实现并行性，并对事件进行窗口化，以便定期发出结果，即使一个键的批处理大小小于批处理大小的事件

    package com.example.dataflow.transform;

    import com.example.dataflow.event.ClickEvent;
    import org.apache.beam.sdk.transforms.DoFn;
    import org.apache.beam.sdk.transforms.GroupIntoBatches;
    import org.apache.beam.sdk.transforms.PTransform;
    import org.apache.beam.sdk.transforms.ParDo;
    import org.apache.beam.sdk.transforms.windowing.FixedWindows;
    import org.apache.beam.sdk.transforms.windowing.Window;
    import org.apache.beam.sdk.values.KV;
    import org.apache.beam.sdk.values.PCollection;
    import org.joda.time.Duration;

    import java.util.Random;

    /**
     * Batch clicks into packs of BATCH_SIZE size
     */
    public class ClickToClicksPack extends PTransform, PCollection>> {
        public static final int BATCH_SIZE = 10;
        // Define window duration.
        // After window's end - elements are emitted even if there are less then BATCH_SIZE elements
        public static final int WINDOW_DURATION_SECONDS = 1;
        private static final int DEFAULT_SHARDS_NUMBER = 20;
        // Determine possible parallelism level
        private int shardsNumber = DEFAULT_SHARDS_NUMBER;

        public ClickToClicksPack() {
            super();
        }

        public ClickToClicksPack(int shardsNumber) {
            super();
            this.shardsNumber = shardsNumber;
        }

        @Override
        public PCollection> expand(PCollection input) {
            return input
                    // assign keys, as "GroupIntoBatches" works only with key-value pairs
                    .apply(ParDo.of(new AssignRandomKeys(shardsNumber)))
                    .apply(Window.into(FixedWindows.of(Duration.standardSeconds(WINDOW_DURATION_SECONDS))))
                    .apply(GroupIntoBatches.ofSize(BATCH_SIZE))
                    .apply(ParDo.of(new ExtractValues()));
        }

        /**
         * Assigns to clicks random integer between zero and shardsNumber
         */
        private static class AssignRandomKeys extends DoFn> {
            private int shardsNumber;
            private Random random;

            AssignRandomKeys(int shardsNumber) {
                super();
                this.shardsNumber = shardsNumber;
            }

            @Setup
            public void setup() {
                random = new Random();
            }

            @ProcessElement
            public void processElement(ProcessContext c) {
                ClickEvent clickEvent = c.element();
                KV kv = KV.of(random.nextInt(shardsNumber), clickEvent);
                c.output(kv);
            }
        }

        /**
         * Extract values from KV
         */
        private static class ExtractValues extends DoFn>, Iterable> {
            @ProcessElement
            public void processElement(ProcessContext c) {
                KV> kv = c.element();
                c.output(kv.getValue());
            }
        }
    }

    package com.example.dataflow.transform;

    import com.example.dataflow.event.ClickEvent;
    import org.apache.beam.sdk.transforms.DoFn;
    import org.apache.beam.sdk.transforms.GroupIntoBatches;
    import org.apache.beam.sdk.transforms.PTransform;
    import org.apache.beam.sdk.transforms.ParDo;
    import org.apache.beam.sdk.transforms.windowing.FixedWindows;
    import org.apache.beam.sdk.transforms.windowing.Window;
    import org.apache.beam.sdk.values.KV;
    import org.apache.beam.sdk.values.PCollection;
    import org.joda.time.Duration;

    import java.util.Random;

    /**
     * Batch clicks into packs of BATCH_SIZE size
     */
    public class ClickToClicksPack extends PTransform, PCollection>> {
        public static final int BATCH_SIZE = 10;
        // Define window duration.
        // After window's end - elements are emitted even if there are less then BATCH_SIZE elements
        public static final int WINDOW_DURATION_SECONDS = 1;
        private static final int DEFAULT_SHARDS_NUMBER = 20;
        // Determine possible parallelism level
        private int shardsNumber = DEFAULT_SHARDS_NUMBER;

        public ClickToClicksPack() {
            super();
        }

        public ClickToClicksPack(int shardsNumber) {
            super();
            this.shardsNumber = shardsNumber;
        }

        @Override
        public PCollection> expand(PCollection input) {
            return input
                    // assign keys, as "GroupIntoBatches" works only with key-value pairs
                    .apply(ParDo.of(new AssignRandomKeys(shardsNumber)))
                    .apply(Window.into(FixedWindows.of(Duration.standardSeconds(WINDOW_DURATION_SECONDS))))
                    .apply(GroupIntoBatches.ofSize(BATCH_SIZE))
                    .apply(ParDo.of(new ExtractValues()));
        }

        /**
         * Assigns to clicks random integer between zero and shardsNumber
         */
        private static class AssignRandomKeys extends DoFn> {
            private int shardsNumber;
            private Random random;

            AssignRandomKeys(int shardsNumber) {
                super();
                this.shardsNumber = shardsNumber;
            }

            @Setup
            public void setup() {
                random = new Random();
            }

            @ProcessElement
            public void processElement(ProcessContext c) {
                ClickEvent clickEvent = c.element();
                KV kv = KV.of(random.nextInt(shardsNumber), clickEvent);
                c.output(kv);
            }
        }

        /**
         * Extract values from KV
         */
        private static class ExtractValues extends DoFn>, Iterable> {
            @ProcessElement
            public void processElement(ProcessContext c) {
                KV> kv = c.element();
                c.output(kv.getValue());
            }
        }
    }