java中多个json到xml的高效处理

我需要处理一个文件中存储的多个json，并将它们转换为xml。这些json平均将转换为3000行xml（以了解json的大小）。 我希望在不到半小时内实现这一目标

这段时间包括从文件中读取、遍历和转换，以及再次以xml的形式存储到另一个文件中。 稍后，我还需要将它们存储在DB中。我还没有这样做，但我计划使用Kafka连接器直接插入数据库以实现性能

我面临的问题是：- -我只能从输入文本文件中读取一行，但不能读取多行。 -如何在接受要转换的字符串的SpliteratorBenchmark.processLine（）方法中使用我的util。 -如何在每次创建新的xml文件时创建新的xml文件。我有一个包含多个json的文件。我应该读一行并将其转换为xml，然后再转换为下一个json的另一个xml

请帮忙

有没有其他方法可以做到这一点。我不能使用MQ或Kafka之类的JMS来完成此任务

这是我的核心固定批处理拆分器数据库：-

public abstract class FixedBatchSpliteratorBase<T> implements Spliterator<T> {
      private final int batchSize;
      private final int characteristics;
      private long est;

      public FixedBatchSpliteratorBase(int characteristics, int batchSize, long est) {
        this.characteristics = characteristics | SUBSIZED;
        this.batchSize = batchSize;
        this.est = est;
      }
      public FixedBatchSpliteratorBase(int characteristics, int batchSize) {
        this(characteristics, batchSize, Long.MAX_VALUE);
      }
      public FixedBatchSpliteratorBase(int characteristics) {
        this(characteristics, 128, Long.MAX_VALUE);
      }
      public FixedBatchSpliteratorBase() {
        this(IMMUTABLE | ORDERED | NONNULL);
      }

      @Override 
      public Spliterator<T> trySplit() {
        final HoldingConsumer<T> holder = new HoldingConsumer<T>();
        if (!tryAdvance(holder)) return null;
        final Object[] a = new Object[batchSize];
        int j = 0;
        do a[j] = holder.value; while (++j < batchSize && tryAdvance(holder));
        if (est != Long.MAX_VALUE) est -= j;
        return spliterator(a, 0, j, characteristics() | SIZED);
      }


      @Override 
      public Comparator<? super T> getComparator() {
        if (hasCharacteristics(SORTED)) return null;
        throw new IllegalStateException();
      }

      @Override 
      public long estimateSize() { 
          return est; 
      }

      @Override 
      public int characteristics() { 
          return characteristics; 
      }

      static final class HoldingConsumer<T> implements Consumer<T> {
        Object value;

        @Override 
        public void accept(T value) { 
            this.value = value; 
        }
      }
    }

---

您所能做的最好的事情就是使用探查器查看CPU时间和内存在哪里花费了大部分可用资源，并在此基础上进行优化

我能够逐个读取多个json文件并创建新的xml文件。 代码如下：-

private static long processLine(Path line) {
        final long localStart = System.nanoTime();
        System.out.println("Path" + line);
        OutputStream outputStream;
        try {
            if(Files.isRegularFile(line)) {
                String str = line.getFileName().toString();
                System.out.println("FIlename - " + str.substring(0, str.length()-4));

                outputStream = new FileOutputStream("/Users/d0s04ub/Documents/STS_Workspace/OrderProcessor/src/main/resources/output/" + str.substring(0, str.length()-4));

                InputStream content = Files.newInputStream(line);

                try (Reader reader = new BufferedReader(new InputStreamReader(content, StandardCharsets.UTF_8)))
                {
                    try {
                        new JsonStreamXMLWriter(reader, new BufferedWriter(new OutputStreamWriter(outputStream))).convert();
                    } catch (XMLStreamException e) {
                        e.printStackTrace();
                    }
                }
            }
        } catch (IOException e1) {
            e1.printStackTrace();
        }finally{

        }


        return System.nanoTime() - localStart;
    }

---

请张贴所有与此相关的实际代码，如果有帮助的话。我最初的猜测是，你可能使用了太多的线程，并导致颠簸，这会使你的速度降低很多。我绝对希望像你所希望的那样，这样的事情可以在30分钟或更短的时间内完成。

import java.util.function.Consumer;

import com.dj.gls.core.FixedBatchSpliteratorBase;


public class JsonSpliterator extends FixedBatchSpliteratorBase<String[]> {
    private final JSONReader jr;

    JsonSpliterator(JSONReader jr, int batchSize) {
      super(IMMUTABLE | ORDERED | NONNULL, batchSize);
      if (jr == null) throw new NullPointerException("JSONReader is null");
      this.jr = jr;
    }
    public JsonSpliterator(JSONReader jr) { this(jr, 128); }

    @Override 
    public boolean tryAdvance(Consumer<? super String[]> action) {
      if (action == null) throw new NullPointerException();
      try {
        final String[] row = jr.readNext();
        if (row == null) return false;
        action.accept(row);
        return true;
      } catch (RuntimeException e) {  
        throw e; 
      } catch (Exception e) { 
        throw new RuntimeException(e); 
      }
    }

    @Override 
    public void forEachRemaining(Consumer<? super String[]> action) {
      if (action == null) throw new NullPointerException();
      try {
        for (String[] row; (row = jr.readNext()) != null;) action.accept(row);
      } catch (RuntimeException e) {  
        throw e; 
      } catch (Exception e) { 
        throw new RuntimeException(e); 
      }
    }
}

import static java.util.concurrent.TimeUnit.SECONDS;

import java.io.IOException;
import java.io.PrintWriter;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.stream.Stream;

import com.dj.gls.core.FixedBatchSpliterator;


public class SpliteratorBenchmark {
    static double sink;

    public static void main(String[] args) throws IOException {
        final Path inputPath = createInput();

        for (int i = 0; i < 3; i++) {
            System.out.println("Start processing JDK stream");
            measureProcessing(Files.lines(inputPath));
            System.out.println("Start processing fixed-batch stream");
            measureProcessing(FixedBatchSpliterator.withBatchSize(Files.lines(inputPath), 10));
        }
    }

    private static void measureProcessing(Stream<String> input) throws IOException {
        final long start = System.nanoTime();
        try (Stream<String> lines = input) {
            final long totalTime = lines.parallel().mapToLong(SpliteratorBenchmark::processLine).sum();
            final double cpuTime = totalTime, realTime = System.nanoTime() - start;
            final int cores = Runtime.getRuntime().availableProcessors();
            System.out.println("          Cores: " + cores);
            System.out.format("       CPU time: %.2f s\n", cpuTime / SECONDS.toNanos(1));
            System.out.format("      Real time: %.2f s\n", realTime / SECONDS.toNanos(1));
            System.out.format("CPU utilization: %.2f%%\n\n", 100.0 * cpuTime / realTime / cores);
        }
    }


    private static long processLine(String line) {
        final long localStart = System.nanoTime();

        //conversion of json to xml and storing an xml file


        return System.nanoTime() - localStart;
    }

    private static Path createInput() throws IOException {
        final Path inputPath = Paths.get("/src/main/resources/input/input.txt");
        return inputPath;
    }
}

import java.io.InputStream;
import java.io.OutputStream;
import java.io.Reader;
import java.io.Writer;
import javax.json.Json;
import javax.json.stream.JsonParser;
import javax.xml.stream.XMLOutputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamWriter;
import javax.xml.transform.Result;

/**
 * Converts JSON stream to XML stream.
 * Uses XML representation of JSON defined in XSLT 3.0.
 * 
 * @see <a href="https://www.w3.org/TR/xslt-30/#json">22 Processing JSON Data</a>
 * 
 */
public class JsonStreamXMLWriter
{

    public static final String XPATH_FUNCTIONS_NS = "http://www.w3.org/2005/xpath-functions";
    private static final XMLOutputFactory XOF = XMLOutputFactory.newInstance();

    static
    {
        XOF.setProperty(XMLOutputFactory.IS_REPAIRING_NAMESPACES, true);
    }

    private final JsonParser parser;
    private final XMLStreamWriter writer;

    public JsonStreamXMLWriter(Reader reader, Writer stream) throws XMLStreamException
    {
        this(Json.createParser(reader), getXMLOutputFactory().createXMLStreamWriter(stream));
    }

    public JsonStreamXMLWriter(Reader reader, OutputStream stream) throws XMLStreamException
    {
        this(Json.createParser(reader), getXMLOutputFactory().createXMLStreamWriter(stream));
    }

    public JsonStreamXMLWriter(Reader reader, OutputStream stream, String encoding) throws XMLStreamException
    {
        this(Json.createParser(reader), getXMLOutputFactory().createXMLStreamWriter(stream, encoding));
    }

    public JsonStreamXMLWriter(Reader reader, Result result) throws XMLStreamException
    {
        this(Json.createParser(reader), getXMLOutputFactory().createXMLStreamWriter(result));
    }

    public JsonStreamXMLWriter(Reader reader, XMLStreamWriter writer)
    {
        this(Json.createParser(reader), writer);
    }

    public JsonStreamXMLWriter(InputStream is, Writer stream) throws XMLStreamException
    {
        this(Json.createParser(is), getXMLOutputFactory().createXMLStreamWriter(stream));
    }

    public JsonStreamXMLWriter(InputStream is, OutputStream stream) throws XMLStreamException
    {
        this(Json.createParser(is), getXMLOutputFactory().createXMLStreamWriter(stream));
    }

    public JsonStreamXMLWriter(InputStream is, OutputStream stream, String encoding) throws XMLStreamException
    {
        this(Json.createParser(is), getXMLOutputFactory().createXMLStreamWriter(stream, encoding));
    }

    public JsonStreamXMLWriter(InputStream is, Result result) throws XMLStreamException
    {
        this(Json.createParser(is), getXMLOutputFactory().createXMLStreamWriter(result));
    }

    public JsonStreamXMLWriter(InputStream is, XMLStreamWriter writer)
    {
        this(Json.createParser(is), writer);
    }

    public JsonStreamXMLWriter(JsonParser parser, Writer stream) throws XMLStreamException
    {
        this(parser, getXMLOutputFactory().createXMLStreamWriter(stream));
    }

    public JsonStreamXMLWriter(JsonParser parser, OutputStream stream) throws XMLStreamException
    {
        this(parser, getXMLOutputFactory().createXMLStreamWriter(stream));
    }

    public JsonStreamXMLWriter(JsonParser parser, OutputStream stream, String encoding) throws XMLStreamException
    {
        this(parser, getXMLOutputFactory().createXMLStreamWriter(stream, encoding));
    }

    public JsonStreamXMLWriter(JsonParser parser, Result result) throws XMLStreamException
    {
        this(parser, getXMLOutputFactory().createXMLStreamWriter(result));
    }

    public JsonStreamXMLWriter(JsonParser parser, XMLStreamWriter writer)
    {
        this.parser = parser;
        this.writer = writer;
    }

    public void convert() throws XMLStreamException
    {
        convert(getWriter());
    }

    public void convert(XMLStreamWriter writer) throws XMLStreamException
    {
        convert(getParser(), writer);
    }

    public static void convert(JsonParser parser, XMLStreamWriter writer) throws XMLStreamException
    {
        writer.writeStartDocument();
        writer.setDefaultNamespace(XPATH_FUNCTIONS_NS);

        write(parser, writer);

        writer.writeEndDocument();
        writer.flush();

        parser.close();
    }

    public static void write(JsonParser parser, XMLStreamWriter writer) throws XMLStreamException
    {
        String keyName = null;
        while (parser.hasNext())
        {
            JsonParser.Event event = parser.next();

            switch(event)
            {
                case START_ARRAY:
                    writer.writeStartElement(XPATH_FUNCTIONS_NS, "array");
                    if (keyName != null)
                    {
                        writer.writeAttribute("key", keyName);
                        keyName = null;
                    }
                break;
                case END_ARRAY:
                    writer.writeEndElement();
                break;
                case START_OBJECT:
                    writer.writeStartElement(XPATH_FUNCTIONS_NS, "map");
                    if (keyName != null)
                    {
                        writer.writeAttribute("key", keyName);
                        keyName = null;
                    }
                break;
                case END_OBJECT:
                    writer.writeEndElement();
                break;
                case VALUE_FALSE:
                    writer.writeStartElement(XPATH_FUNCTIONS_NS, "boolean");
                    if (keyName != null)
                    {
                        writer.writeAttribute("key", keyName);
                        keyName = null;
                    }
                    writer.writeCharacters("false");
                    writer.writeEndElement();
                break;
                case VALUE_TRUE:
                    writer.writeStartElement(XPATH_FUNCTIONS_NS, "boolean");
                    if (keyName != null)
                    {
                        writer.writeAttribute("key", keyName);
                        keyName = null;
                    }
                    writer.writeCharacters("true");
                    writer.writeEndElement();
                break;
                case KEY_NAME:
                    keyName = parser.getString();
                break;
                case VALUE_STRING:
                    writer.writeStartElement(XPATH_FUNCTIONS_NS, "string");
                    if (keyName != null)
                    {
                        writer.writeAttribute("key", keyName);
                        keyName = null;
                    }
                    writer.writeCharacters(parser.getString());
                    writer.writeEndElement();
                break;
                case VALUE_NUMBER:
                    writer.writeStartElement(XPATH_FUNCTIONS_NS, "number");
                    if (keyName != null)
                    {
                        writer.writeAttribute("key", keyName);
                        keyName = null;
                    }
                    writer.writeCharacters(parser.getString());
                    writer.writeEndElement();
                break;
                case VALUE_NULL:
                    writer.writeEmptyElement(XPATH_FUNCTIONS_NS, "null");
                    if (keyName != null)
                    {
                        writer.writeAttribute("key", keyName);
                        keyName = null;
                    }
                break;
            }

            writer.flush();
        }
    }

    protected JsonParser getParser()
    {
        return parser;
    }

    protected XMLStreamWriter getWriter()
    {
        return writer;
    }

    protected static XMLOutputFactory getXMLOutputFactory()
    {
        return XOF;
    }

}

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.nio.charset.StandardCharsets;
import javax.xml.stream.XMLStreamException;

import org.apache.commons.io.IOUtils;


public class JSON2XML
{

    public static void main(String[] args) throws IOException, XMLStreamException
    {
        OutputStreamWriter outputStreamWriter = new OutputStreamWriter(outputStream);

        InputStream json = new FileInputStream(file);

        if (json.available() == 0)
        {
            System.err.println("JSON input: stdin");
            System.exit(1);
        }

        try (Reader reader = new BufferedReader(new InputStreamReader(json, StandardCharsets.UTF_8)))
        {

            new JsonStreamXMLWriter(reader, new BufferedWriter(new OutputStreamWriter(outputStream))).convert();
        }
    }

}

private static long processLine(Path line) {
        final long localStart = System.nanoTime();
        System.out.println("Path" + line);
        OutputStream outputStream;
        try {
            if(Files.isRegularFile(line)) {
                String str = line.getFileName().toString();
                System.out.println("FIlename - " + str.substring(0, str.length()-4));

                outputStream = new FileOutputStream("/Users/d0s04ub/Documents/STS_Workspace/OrderProcessor/src/main/resources/output/" + str.substring(0, str.length()-4));

                InputStream content = Files.newInputStream(line);

                try (Reader reader = new BufferedReader(new InputStreamReader(content, StandardCharsets.UTF_8)))
                {
                    try {
                        new JsonStreamXMLWriter(reader, new BufferedWriter(new OutputStreamWriter(outputStream))).convert();
                    } catch (XMLStreamException e) {
                        e.printStackTrace();
                    }
                }
            }
        } catch (IOException e1) {
            e1.printStackTrace();
        }finally{

        }


        return System.nanoTime() - localStart;
    }