Neo4j错误：java.lang.OutOfMemoryError:超出GC开销限制
我使用Neo4j错误：java.lang.OutOfMemoryError:超出GC开销限制,java,neo4j,cypher,Java,Neo4j,Cypher,我使用StatementResult=session.run（Cypher\u查询）使用java运行Cypher查询。所涉及的程序用于从csv文件中导入带有LOAD csv 将引发此错误： Exception in thread "main" java.lang.OutOfMemoryError: GC overhead limit exceeded at java.util.logging.SimpleFormatter.format(Unknown Source) at java.util.
StatementResult=session.run（Cypher\u查询）LOAD csvException in thread "main" java.lang.OutOfMemoryError: GC overhead limit exceeded
at java.util.logging.SimpleFormatter.format(Unknown Source)
at java.util.logging.StreamHandler.publish(Unknown Source)
at java.util.logging.ConsoleHandler.publish(Unknown Source)
at java.util.logging.Logger.log(Unknown Source)
at java.util.logging.Logger.doLog(Unknown Source)
at java.util.logging.Logger.log(Unknown Source)
at org.neo4j.driver.internal.logging.JULogger.warn(JULogger.java:54)
at org.neo4j.driver.internal.security.TLSSocketChannel.close(TLSSocketChannel.java:477)
at org.neo4j.driver.internal.net.SocketClient.stop(SocketClient.java:192)
at org.neo4j.driver.internal.net.SocketConnection.close(SocketConnection.java:260)
at org.neo4j.driver.internal.net.ConcurrencyGuardingConnection.close(ConcurrencyGuardingConnection.java:178)
at org.neo4j.driver.internal.net.pooling.PooledSocketConnection.dispose(PooledSocketConnection.java:251)
at org.neo4j.driver.internal.net.pooling.PooledConnectionReleaseConsumer.accept(PooledConnectionReleaseConsumer.java:50)
at org.neo4j.driver.internal.net.pooling.PooledConnectionReleaseConsumer.accept(PooledConnectionReleaseConsumer.java:29)
at org.neo4j.driver.internal.net.pooling.PooledSocketConnection.close(PooledSocketConnection.java:200)
at org.neo4j.driver.internal.NetworkSession.closeCurrentConnection(NetworkSession.java:385)
at org.neo4j.driver.internal.NetworkSession.syncAndCloseCurrentConnection(NetworkSession.java:359)
at org.neo4j.driver.internal.NetworkSession.run(NetworkSession.java:102)
at org.neo4j.driver.internal.NetworkSession.run(NetworkSession.java:93)
at org.neo4j.driver.internal.NetworkSession.run(NetworkSession.java:73)
import java.io.File;
import java.io.FileNotFoundException;
import org.neo4j.driver.v1.AuthTokens;
import org.neo4j.driver.v1.Driver;
import org.neo4j.driver.v1.GraphDatabase;
import org.neo4j.driver.v1.Record;
import org.neo4j.driver.v1.Session;
import org.neo4j.driver.v1.StatementResult;
import java.util.List;
import java.util.Scanner;
import java.util.logging.Logger;
public class ImporterImprovedVersion {
private static String path = "file:///C:/csvsToImport/csvs";
private int num;
private boolean headerPresent(String filePath) throws FileNotFoundException{
Scanner scanner = new Scanner(new File(filePath));
if(scanner.hasNext()){
String firstLine = scanner.nextLine();
scanner.close();
if(filePath.contains("address")){
System.out.println("address! " + this.num + " " + firstLine.contains("addr_tag_links"));
return firstLine.contains("addr_tag_links");
}else if (filePath.contains("transaction")){
System.out.println("transaction! " + this.num + " " + firstLine.contains("value_bitcoin"));
return firstLine.contains("value_bitcoin");
}else{
System.out.println("wallet! " + this.num + " " + firstLine.contains("primAddress,firstSeenTime"));
return firstLine.contains("primAddress,firstSeenTime");
}
}
scanner.close();
return false;
}
public ImporterImprovedVersion(int num, boolean withOutputIndex, Session session, String mainDir) throws FileNotFoundException{
this.num = num;
//import csvs and create corresponding nodes
String folderPath = ImporterImprovedVersion.path + num + "/";
System.out.println(folderPath);
String queryAddr = null;
if(this.headerPresent(mainDir + num + "\\addresses.csv")){
queryAddr = "LOAD CSV From '" + folderPath + "addresses.csv' AS line"
+ " WITH line"
+ " SKIP 1";
}else{
queryAddr = "LOAD CSV From '" + folderPath + "addresses.csv' AS line";
}
String queryWallet = null;
if(this.headerPresent(mainDir + num + "\\wallet.csv")){
queryWallet = "LOAD CSV From '" + folderPath + "wallet.csv' AS line"
+ " WITH line"
+ " SKIP 1";
}else{
queryWallet = "LOAD CSV From '" + folderPath + "wallet.csv' AS line";
}
File tranFile = new File(mainDir + num + "\\transactionRelation.csv");
String queryTran = null;
System.out.println(tranFile);
System.out.println(tranFile.exists());
if(tranFile.exists()){
if(this.headerPresent(mainDir + num + "\\transactionRelation.csv")){
queryTran = "LOAD CSV From '" + folderPath + "transactionRelation.csv' AS line"
+ " WITH line"
+ " SKIP 1";
}else{
queryTran = "LOAD CSV From '" + folderPath + "transactionRelation.csv' AS line";
}
}else{
if(this.headerPresent(mainDir + num + "\\transactionRelation1.csv")){
queryTran = "LOAD CSV From '" + folderPath + "transactionRelation1.csv' AS line"
+ " WITH line"
+ " SKIP 1";
}else{
queryTran = "LOAD CSV From '" + folderPath + "transactionRelation1.csv' AS line";
}
}
StatementResult sR = session.run(queryWallet
+ " MERGE (w:Wallet { primWallAddr:line[0]})"
+ " ON CREATE SET w.first_seen = line[1], w.last_seen=line[2]");
System.out.println("Wallet nodes created");
sR = session.run(queryAddr + " MERGE (a:Address {AddId:(line[0])})"
+ " SET a.addr_tag_link= CASE WHEN a.addr_tag_link = 'null' or a.addr_tag_link IS NULL THEN line[1] ELSE a.addr_tag_link END,"
+ " a.addr_tag= CASE WHEN a.addr_tag = 'null' or a.addr_tag IS NULL THEN line[2] ELSE a.addr_tag END,"
+ " a.first_seen=line[3], a.last_seen=line[4], a.multiExist= a.multiExist OR apoc.convert.toBoolean(line[6])"
+ " WITH a, line[5] as li5"
+ " MATCH (a), (wa:Wallet) WHERE li5=wa.primWallAddr"
+ " MERGE (a)-[r:BelongTo{uniqueReferenceBelongTo:(a.AddId + wa.primWallAddr)}]->(wa)"
+ " ON CREATE SET r.primWallAddr = wa.primWallAddr,"
+ " wa.first_seen = CASE WHEN apoc.date.parse(wa.first_seen, 's',\"yyyy-MM-dd'T'HH:mm:ss\") < apoc.date.parse(a.first_seen, 's',\"yyyy-MM-dd'T'HH:mm:ss\") THEN wa.first_seen ELSE a.first_seen END,"
+ " wa.last_seen = CASE WHEN apoc.date.parse(wa.last_seen, 's',\"yyyy-MM-dd'T'HH:mm:ss\") > apoc.date.parse(a.last_seen, 's',\"yyyy-MM-dd'T'HH:mm:ss\") THEN wa.last_seen ELSE a.last_seen END");
System.out.println("Address nodes created");
System.out.println("BelongTo rel created");
if(withOutputIndex){
sR = session.run(queryTran
+ " MATCH (senderAddress:Address {AddId:line[0]}), (senderAddress)-[:BelongTo]->(sender:Wallet),"
+ " (receiverAddress:Address {AddId:line[1]}), (receiverAddress)-[:BelongTo]->(receiver:Wallet)"
+ " MERGE (sender)-[r:SendTo{uniqueReferenceTran:(line[2] + line[8])}]->(receiver)"
+ " ON CREATE SET r.tranHashString=line[2],r.time=line[3],r.value_bitcoin=line[4],"
+ "r.value_dollar=line[5],r.type=line[6],r.estChanAddr=line[7],r.outputIndex=line[8]");
}else{
System.out.println("withoutOutputIndex");
sR = session.run(queryTran
+ " MATCH (senderAddress:Address {AddId:line[0]}), (senderAddress)-[:BelongTo]->(sender:Wallet),"
+ " (receiverAddress:Address {AddId:line[1]}), (receiverAddress)-[:BelongTo]->(receiver:Wallet)"
+ " MERGE (sender)-[r:SendTo{uniqueReferenceTran:(line[2] + 'default')}]->(receiver)"
+ " ON CREATE SET r.tranHashString=line[2],r.time=line[3],r.value_bitcoin=line[4],"
+ " r.value_dollar=line[5],r.type=line[6],r.estChanAddr=line[7],r.outputIndex='default'");
}
sR.list();
System.out.println("Tran rel created");
String queryAddrDiffWallet = "MATCH (a:Address)-[:BelongTo]->(w1:Wallet), (a)-[r0:BelongTo]->(w2:Wallet)"
+ " WHERE w1 <> w2"
+ " RETURN count(a)";
sR = session.run(queryAddrDiffWallet);
List<Record> records = sR.list();
System.out.println(records.get(0).get("count(a)").asInt());
System.out.println(records.get(0).get("count(a)").asInt() == 0);
if(records.get(0).get("count(a)").asInt() != 0){
sR = session.run("MATCH (a:Address)-[:BelongTo]->(w1:Wallet), (a)-[r0:BelongTo]->(w2:Wallet)"
+ " WHERE w1 <> w2"
+ " RETURN a");
}
String queryMergeWallet = "MATCH (a:Address)-[:BelongTo]->(w1:Wallet)"
+ " WITH DISTINCT a, min(ID(w1)) as minId"
+ " MATCH (minW:Wallet)"
+ " WHERE ID(minW) = minId"
+ " SET a.primWallAddr = minW.primWallAddr"
+ " WITH DISTINCT minW, a"
+ " MATCH (a)-[r0:BelongTo]->(w2:Wallet)"
+ " WHERE minW <> w2"
+ " WITH DISTINCT minW, w2, r0"
+ " SET minW.first_seen = CASE WHEN apoc.date.parse(minW.first_seen, 's',\"yyyy-MM-dd'T'HH:mm:ss\") < apoc.date.parse(w2.first_seen, 's',\"yyyy-MM-dd'T'HH:mm:ss\") THEN minW.first_seen ELSE w2.first_seen END,"
+ " minW.last_seen = CASE WHEN apoc.date.parse(minW.last_seen, 's',\"yyyy-MM-dd'T'HH:mm:ss\") > apoc.date.parse(w2.last_seen, 's',\"yyyy-MM-dd'T'HH:mm:ss\") THEN minW.last_seen ELSE w2.last_seen END"
+ " WITH DISTINCT minW, w2, r0"
+ " DELETE r0"
+ " WITH DISTINCT minW, w2"
+ " MATCH (b:Address)-[r:BelongTo]->(w2)"
+ " WITH DISTINCT r, minW, w2, b"
+ " DELETE r"
+ " WITH DISTINCT b, minW, w2"
+ " MERGE (b)-[be:BelongTo{uniqueReferenceBelongTo:(b.AddId + minW.primWallAddr)}]->(minW)"
+ " ON CREATE SET be.primWallAddr = minW.primWallAddr"
+ " WITH DISTINCT w2, minW"
+ " MATCH (w3:Wallet)-[r2:SendTo]->(w2)"
+ " MERGE (w3)-[rN:SendTo{uniqueReferenceTran:r2.uniqueReferenceTran}]->(minW)"
+ " ON CREATE SET rN.tranHashString=r2.tranHashString,rN.time=r2.time,"
+ "rN.value_bitcoin=r2.value_bitcoin,rN.value_dollar=r2.value_dollar,"
+ "rN.type=r2.type,rN.estChanAddr=r2.estChanAddr,rN.outputIndex=r2.outputIndex"
+ " WITH DISTINCT w2, minW, r2"
+ " DELETE r2"
+ " WITH DISTINCT w2, minW"
+ " MATCH (w2)-[r1:SendTo]->(w4:Wallet)"
+ " MERGE (minW)-[rN2:SendTo{uniqueReferenceTran:r1.uniqueReferenceTran}]->(w4)"
+ " ON CREATE SET rN2.tranHashString=r1.tranHashString,rN2.time=r1.time,"
+ "rN2.value_bitcoin=r1.value_bitcoin,rN2.value_dollar=r1.value_dollar,"
+ "rN2.type=r1.type,rN2.estChanAddr=r1.estChanAddr,rN2.outputIndex=r1.outputIndex"
+ " WITH DISTINCT r1"
+ " DELETE r1"
;
if(records.get(0).get("count(a)").asInt() != 0){
//merge any wallet containing same addresses and adjust the transaction as well (w2 merge to w1)
try{
sR = session.run(queryMergeWallet);
}catch(Exception e){
e.printStackTrace();
try {
Thread.sleep(20000);
} catch (InterruptedException e1) {
e1.printStackTrace();
}
sR = session.run(queryMergeWallet);
}
}
System.out.println("same wallet merged");
System.out.println("----------abc----------------------" + num);
//delete all wallets with no addresses
sR = session.run(
" MATCH (n:Wallet)"
+ " WHERE NOT ()-[:BelongTo]->(n)"
+ " DETACH DELETE n"
);
System.out.println("Wall with no addr deleted");
}
public static void main(String[] args) throws FileNotFoundException{
Driver driver = GraphDatabase.driver( "bolt://localhost:7687", AuthTokens.basic( "neo4j", "neo7474" ) );
try(Session session = driver.session()){
String mainDir = "C:\\Program Files\\neo4j-enterprise-3.1.1\\import\\csvsToImport\\csvs";
// create index
session.run( "CREATE CONSTRAINT ON (addr:Address) ASSERT addr.AddId IS UNIQUE");
session.run( "CREATE CONSTRAINT ON (wa:Wallet) ASSERT wa.primWallAddr IS UNIQUE");
System.out.println("aaa");
int counter = 537;
while(counter < 660){
File dir = new File(mainDir + counter);
if(dir.exists()){
System.out.println("aaa");
ImporterImprovedVersion a = new ImporterImprovedVersion(counter, counter > 260, session, mainDir);
}
counter ++;
}
}
}
}
导入java.io.File；
导入java.io.FileNotFoundException；
导入org.neo4j.driver.v1.AuthTokens；
导入org.neo4j.driver.v1.driver；
导入org.neo4j.driver.v1.GraphDatabase；
导入org.neo4j.driver.v1.Record；
导入org.neo4j.driver.v1.Session；
导入org.neo4j.driver.v1.StatementResult；
导入java.util.List；
导入java.util.Scanner；
导入java.util.logging.Logger；
公共类导入改进版本{
专用静态字符串路径=”file:///C:/csvsToImport/csvs";
私有整数；
私有布尔headerPresent（字符串文件路径）引发FileNotFoundException{
Scanner Scanner=新扫描仪（新文件（文件路径））；
if（scanner.hasNext（））{
字符串firstLine=scanner.nextLine（）；
scanner.close（）；
if（filePath.contains（“地址”））{
System.out.println（“address！”+this.num+“”+firstLine.contains（“addr\u tag\u links”）；
返回firstLine.contains（“addr_tag_links”）；
}else if（filePath.contains（“事务”））{
System.out.println（“transaction！”+this.num+“”+firstLine.contains（“value_比特币”）；
返回firstLine.contains（“比特币价值”）；
}否则{
System.out.println（“wallet！”+this.num+“”+firstLine.contains（“primAddress，firstSeenTime”）；
返回firstLine.contains（“primAddress，firstSeenTime”）；
}
}
scanner.close（）；
返回false；
}
public ImporterImprovedVersion（int num，boolean withOutputIndex，Session Session，String mainDir）抛出FileNotFoundException{
this.num=num；
//导入CSV并创建相应的节点
字符串folderPath=ImporterImprovedVersion.path+num+“/”；
System.out.println（folderPath）；
字符串queryAddr=null；
if（此.headerPresent（mainDir+num+“\\addresses.csv”））{
queryAddr=“将CSV从“+”文件夹路径“+”地址加载到.CSV”作为行”
+“带行”
+“跳过1”；
}否则{
queryAddr=“将CSV从“+”文件夹路径“+”地址.CSV”加载为行”；
}
字符串queryWallet=null；
如果（此.headerPresent（mainDir+num+“\\wallet.csv”））{
queryWallet=“将CSV从“”+folderPath+“wallet.CSV”加载为行”
+“带行”
+“跳过1”；
}否则{
queryWallet=“将CSV从“”+folderPath+“wallet.CSV”加载为行”；
}
File tranFile=新文件（mainDir+num+“\\transactionRelation.csv”）；
字符串queryTran=null；
System.out.println（tranFile）；
System.out.println（tranFile.exists（））；
如果（tranFile.exists（））{
if（this.headerPresent（mainDir+num+“\\transactionRelation.csv”））{
queryTran=“将CSV从“”+folderPath+“transactionRelation.CSV”加载为行”
+“带行”
+“跳过1”；
}否则{
queryTran=“将CSV从“”+folderPath+“transactionRelation.CSV”加载为行”；
}
}否则{
if（此.headerPresent（mainDir+num+“\\transactionRelation1.csv”））{
queryTran=“将CSV从“”+folderPath+“transactionRelation1.CSV”加载为行”
+“带行”
+“跳过1”；
}否则{
queryTran=“将CSV从“”+folderPath+“transactionRelation1.CSV”加载为行”；
}
}
StatementResult sR=session.run（queryWallet
+“合并（w:Wallet{primWallAddr:line[0]}）”
+“在创建集合时，w.first\u seen=行[1]，w.last\u seen=行[2]”；
System.out.println（“创建的钱包节点”）；
sR=session.run（queryAddr+“MERGE（a:Address{AddId:[0]））
+当a.addr\u tag\u link='null'或a.addr\u tag\u link为null时，设置a.addr\u tag\u link=CASE，然后第[1]行，否则a.addr\u tag\u link结束
+“a.addr_标记=当a.addr_标记='null'或a.addr_标记为null时，则第[2]行，否则a.addr_标记结束，”
+“a.first\u seen=行[3]，a.last\u seen=行[4]，a.multiExist=a.multiExist或apoc.convert.toBoolean（行[6]）”
+带一个，行[5]为li5
+匹配（a），（wa:Wallet），其中li5=wa.primWallAddr
+“合并（a）-[r:BelongTo{uniqueReferenceBelongTo:（a.AddId+wa.primWallAddr）}]->（wa）”
+在创建集r.primWallAddr=wa.primWallAddr时
+“wa.first\u seen=apoc.date.parse（wa.first\u seen's'，\“yyyy-MM-dd'HH:MM:ss\”apoc.date.parse（a.last_seen，‘s’，\“yyyy-MM-dd'T'HH:MM:ss\”）时的大小写，然后wa.last_seen-dd'HH:MM:ss\”；
System.out.println（“创建的地址节点”）；
System.out.println（“BelongTo rel created”）；
如果（不带输出索引）{
sR=会话.run（queryTran
+匹配（发件人地址：地址{AddId:line[0]}），（发件人地址）-[：BelongTo]->（发件人：钱包），“
+（接收方地址：地址{AddId:line[1]}），（接收方地址）-[：BelongTo]->（接收方：钱包）
+“合并（发送方）-[r:SendTo{uniqueReferenceTran:（第[2]行+第[8]行]}]->（接收
USING PERIODIC COMMIT 10000
LOAD CSV WITH HEADERS FROM "file:///customers.csv" AS row
CREATE (:Customer {companyName: row.CompanyName})