Hadoop 无法将现有文件附加到HDFS
我在VM上运行单节点Hadoop 1.2.1集群 我的hdfs-site.xml如下所示：Hadoop 无法将现有文件附加到HDFS,hadoop,bigdata,Hadoop,Bigdata,我在VM上运行单节点Hadoop 1.2.1集群 我的hdfs-site.xml如下所示： <configuration> <property> <name>dfs.replication</name> <value>1</value> <description>Default block replication. </description> </property> <
<configuration>
<property>
<name>dfs.replication</name>
<value>1</value>
<description>Default block replication.
</description>
</property>
<property>
<name>dfs.support.append</name>
<value>true</value>
<description>Does HDFS allow appends to files?
</description>
</property>
</configuration>
org.apache.hadoop.ipc.RemoteException: java.io.IOException: Append is not supported. Please see the dfs.support.append configuration parameter
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.appendFile(FSNamesystem.java:1781)
at org.apache.hadoop.hdfs.server.namenode.NameNode.append(NameNode.java:725)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:587)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1432)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1428)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:415)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1190)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1426)
at org.apache.hadoop.ipc.Client.call(Client.java:1113)
at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:229)
at com.sun.proxy.$Proxy1.append(Unknown Source)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:85)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:62)
at com.sun.proxy.$Proxy1.append(Unknown Source)
at org.apache.hadoop.hdfs.DFSClient.append(DFSClient.java:933)
at org.apache.hadoop.hdfs.DFSClient.append(DFSClient.java:922)
at org.apache.hadoop.hdfs.DistributedFileSystem.append(DistributedFileSystem.java:196)
at org.apache.hadoop.fs.FileSystem.append(FileSystem.java:659)
at com.vanilla.hadoop.AppendToHdfsFile.main(AppendToHdfsFile.java:29)
怎么了？我遗漏了什么吗？您应该尝试使用2.X.X版本或0.2X版本，因为在hadoop 0.20.2之后在hdfs上追加文件。请参阅和的详细信息。您应该尝试使用2.X.X版本或0.2X版本，因为在hadoop 0.20.2之后在hdfs上追加文件。有关更多信息，请参见和自1.0.3起不支持追加。无论如何，如果您确实需要前面的功能，要启用附加功能，请将标志“dfs.support.break.append”设置为true
从1.0.3开始不支持追加。无论如何，如果您确实需要前面的功能，要启用附加功能，请将标志“dfs.support.break.append”设置为true
现在让我们从配置文件系统开始：
public FileSystem configureFileSystem(String coreSitePath, String hdfsSitePath) {
FileSystem fileSystem = null;
try {
Configuration conf = new Configuration();
conf.setBoolean("dfs.support.append", true);
Path coreSite = new Path(coreSitePath);
Path hdfsSite = new Path(hdfsSitePath);
conf.addResource(coreSite);
conf.addResource(hdfsSite);
fileSystem = FileSystem.get(conf);
} catch (IOException ex) {
System.out.println("Error occurred while configuring FileSystem");
}
return fileSystem;
}
public String appendToFile(FileSystem fileSystem, String content, String dest) throws IOException {
Path destPath = new Path(dest);
if (!fileSystem.exists(destPath)) {
System.err.println("File doesn't exist");
return "Failure";
}
Boolean isAppendable = Boolean.valueOf(fileSystem.getConf().get("dfs.support.append"));
if(isAppendable) {
FSDataOutputStream fs_append = fileSystem.append(destPath);
PrintWriter writer = new PrintWriter(fs_append);
writer.append(content);
writer.flush();
fs_append.hflush();
writer.close();
fs_append.close();
return "Success";
}
else {
System.err.println("Please set the dfs.support.append property to true");
return "Failure";
}
}
public String readFromHdfs(FileSystem fileSystem, String hdfsFilePath) {
Path hdfsPath = new Path(hdfsFilePath);
StringBuilder fileContent = new StringBuilder("");
try{
BufferedReader bfr=new BufferedReader(new InputStreamReader(fileSystem.open(hdfsPath)));
String str;
while ((str = bfr.readLine()) != null) {
fileContent.append(str+"\n");
}
}
catch (IOException ex){
System.out.println("----------Could not read from HDFS---------\n");
}
return fileContent.toString();
}
public void closeFileSystem(FileSystem fileSystem){
try {
fileSystem.close();
}
catch (IOException ex){
System.out.println("----------Could not close the FileSystem----------");
}
}
要获得完整的参考信息，请使用让我们从配置文件系统开始：
public FileSystem configureFileSystem(String coreSitePath, String hdfsSitePath) {
FileSystem fileSystem = null;
try {
Configuration conf = new Configuration();
conf.setBoolean("dfs.support.append", true);
Path coreSite = new Path(coreSitePath);
Path hdfsSite = new Path(hdfsSitePath);
conf.addResource(coreSite);
conf.addResource(hdfsSite);
fileSystem = FileSystem.get(conf);
} catch (IOException ex) {
System.out.println("Error occurred while configuring FileSystem");
}
return fileSystem;
}
public String appendToFile(FileSystem fileSystem, String content, String dest) throws IOException {
Path destPath = new Path(dest);
if (!fileSystem.exists(destPath)) {
System.err.println("File doesn't exist");
return "Failure";
}
Boolean isAppendable = Boolean.valueOf(fileSystem.getConf().get("dfs.support.append"));
if(isAppendable) {
FSDataOutputStream fs_append = fileSystem.append(destPath);
PrintWriter writer = new PrintWriter(fs_append);
writer.append(content);
writer.flush();
fs_append.hflush();
writer.close();
fs_append.close();
return "Success";
}
else {
System.err.println("Please set the dfs.support.append property to true");
return "Failure";
}
}
public String readFromHdfs(FileSystem fileSystem, String hdfsFilePath) {
Path hdfsPath = new Path(hdfsFilePath);
StringBuilder fileContent = new StringBuilder("");
try{
BufferedReader bfr=new BufferedReader(new InputStreamReader(fileSystem.open(hdfsPath)));
String str;
while ((str = bfr.readLine()) != null) {
fileContent.append(str+"\n");
}
}
catch (IOException ex){
System.out.println("----------Could not read from HDFS---------\n");
}
return fileContent.toString();
}
public void closeFileSystem(FileSystem fileSystem){
try {
fileSystem.close();
}
catch (IOException ex){
System.out.println("----------Could not close the FileSystem----------");
}
}