Java 属性[项目]标记为相互矛盾的注释
我的要求是从谷歌云存储中读取csv文件,并将其加载到谷歌数据存储中。下面是代码片段Java 属性[项目]标记为相互矛盾的注释,java,maven,google-cloud-platform,google-cloud-datastore,google-cloud-storage,Java,Maven,Google Cloud Platform,Google Cloud Datastore,Google Cloud Storage,我的要求是从谷歌云存储中读取csv文件,并将其加载到谷歌数据存储中。下面是代码片段 import com.google.datastore.v1.Entity; import com.google.datastore.v1.Key; import com.opencsv.CSVParser; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.io.TextIO; import org.apache.beam.sdk
import com.google.datastore.v1.Entity;
import com.google.datastore.v1.Key;
import com.opencsv.CSVParser;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.io.TextIO;
import org.apache.beam.sdk.io.gcp.datastore.DatastoreIO;
import org.apache.beam.sdk.options.Default;
import org.apache.beam.sdk.options.Description;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.ParDo;
import org.slf4j.LoggerFactory;
import org.slf4j.Logger;
import javax.annotation.Nullable;
import java.util.UUID;
import static com.google.datastore.v1.client.DatastoreHelper.makeKey;
import static
com.google.datastore.v1.client.DatastoreHelper.makeValue;
public class PipelineClass {
static class CreateEntitiesFn extends DoFn<String, Entity> {
/**
*
*/
private static final Logger LOG = LoggerFactory.getLogger(PipelineClass.class);
private static final long serialVersionUID = 1L;
private final String namespace;
private final String kind;
private final Key ancestorKey;
CreateEntitiesFn(String namespace, String kind) {
this.namespace = namespace;
this.kind = kind;
ancestorKey = makeAncestorKey(namespace, kind);
}
Entity makeEntity(String id, String group) {
Entity.Builder entityBuilder = Entity.newBuilder();
Key.Builder keyBuilder = makeKey(ancestorKey, kind,
UUID.randomUUID().toString());
if (namespace != null) {
keyBuilder.getPartitionIdBuilder().setNamespaceId(namespace);
}
entityBuilder.setKey(keyBuilder.build());
entityBuilder.getMutableProperties().put("id",
makeValue(id).build());
entityBuilder.getMutableProperties().put("group",
makeValue(group).build());
return entityBuilder.build();
}
public void processElement(ProcessContext c) throws Exception {
CSVParser parser = new CSVParser();
String[] parts = parser.parseLine(c.element());
String id = parts[0];
String group = parts[1];
c.output(makeEntity(id, group));
}
}
static Key makeAncestorKey(@Nullable String namespace, String kind) {
Key.Builder keyBuilder = makeKey(kind, "root");
if (namespace != null) {
keyBuilder.getPartitionIdBuilder().setNamespaceId(namespace);
}
return keyBuilder.build();
}
public interface Options extends PipelineOptions {
@Description("Path of the file to read from and store to Cloud Datastore")
@Default.String("gs://myproject-263315.appspot.com/source/food.csv")
String getInput();
void setInput(String value);
@Description("Dataset ID to read from Cloud Datastore")
@Default.String("myproject-263315")
String getProject();
void setProject(String value);
@Description("Cloud Datastore Entity Kind")
@Default.String("FoodGroup")
String getKind();
void setKind(String value);
@Description("Dataset namespace")
@Default.String("nutrients")
String getNamespace();
void setNamespace(@Nullable String value);
@Description("Number of output shards")
@Default.Integer(0)
int getNumShards();
void setNumShards(int value);
}
public static void main(String args[]) {
// PipelineOptionsFactory.register(Options.class);
Options options =
PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);
Pipeline p = Pipeline.create(options);
p.apply(TextIO.read().from(options.getInput()))
.apply(ParDo.of(new
CreateEntitiesFn(options.getNamespace(), options.getKind())))
.apply(DatastoreIO.v1().write().withProjectId(options.getProject()));
p.run();
}
}
我得到了bleow错误
Exception in thread "main" java.lang.IllegalArgumentException: Property [project] is marked with contradictory annotations. Found [[Default.String(value=myproject-263315) on my.dataflow1.PipelineClass$Options#getProject()], [Default.InstanceFactory(value=class org.apache.beam.sdk.extensions.gcp.options.GcpOptions$DefaultProjectFactory) on org.apache.beam.runners.dataflow.options.DataflowPipelineOptions#getProject()], [Default.InstanceFactory(value=class org.apache.beam.sdk.extensions.gcp.options.GcpOptions$DefaultProjectFactory) on org.apache.beam.sdk.extensions.gcp.options.GcpOptions#getProject()]].
有人能帮我纠正一下吗。在这件事上坚持了很长时间。非常感谢您的帮助
谢谢。选项“项目”是PiplineOptions中的保留选项。“JobName”是另一个。我发现了这个,可能对你有用。它有一个Java解决方案和一个简单的Python脚本。希望这有帮助!如果保留了该选项,会发生什么情况?无法在
管道选项工厂中设置值
?我试图设置选项region
(ApacheBeam 2.21.0),但我得到了相同的错误:属性[region]标记了相互矛盾的注释。在org.apache.beam.runners.DataflowPipelineOptions#getRegion()上找到[[Default.String(value=europe-west1)],[Default.InstanceFactory(value=class org.apache.beam.runners.options.DataflowPipelineOptions$DefaultGcpRegionFactory)]
mvn compile exec:java -e -Dexec.mainClass=com.dataflow1.PipelineClass -Dexec.args="--project=myproject-263315 --runner=DataflowRunner" -Pdataflow-runner
Exception in thread "main" java.lang.IllegalArgumentException: Property [project] is marked with contradictory annotations. Found [[Default.String(value=myproject-263315) on my.dataflow1.PipelineClass$Options#getProject()], [Default.InstanceFactory(value=class org.apache.beam.sdk.extensions.gcp.options.GcpOptions$DefaultProjectFactory) on org.apache.beam.runners.dataflow.options.DataflowPipelineOptions#getProject()], [Default.InstanceFactory(value=class org.apache.beam.sdk.extensions.gcp.options.GcpOptions$DefaultProjectFactory) on org.apache.beam.sdk.extensions.gcp.options.GcpOptions#getProject()]].