有没有Spring支持wedhdfs?我没有在谷歌上找到任何有用的链接.
我想通过webhdfs连接到具有正常身份验证和kerberos身份验证的hadoop.春天是否支持这个?
任何有用的链接都会有所帮助.
谢谢
是的,Spring Data支持这一点.根据此文档,可以配置任何受支持的Hadoop文件系统:
http://docs.spring.io/spring-hadoop/docs/current/reference/html/fs.html
SHDP不强制使用任何特定协议 - 实际上,如本节所述,
FileSystem
可以使用任何实现,甚至允许使用除HDFS之外的其他实现.
请参阅下面的代码示例,演示如何将WebHDFS FileSystem
实例自动连接到命令行应用程序.要运行它,请将文件路径作为命令行参数传递,它将通过调用列出该路径中存在的每个文件FileSystem.listStatus
.
代码示例配置为使用"简单"身份验证连接到不安全的WebHDFS实例.要连接到使用Kerberos保护的WebHDFS实例,您需要在<hdp:configuration id="hadoopConfiguration" />
bean中设置相关的配置属性.Hadoop安全配置是一个非常大的主题.而不是重复信息,我只是指向Apache中的文档:
http://hadoop.apache.org/docs/r2.4.1/hadoop-project-dist/hadoop-common/SecureMode.html
<?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> <modelVersion>4.0.0</modelVersion> <groupId>test-spring-hadoop</groupId> <artifactId>test-webhdfs</artifactId> <packaging>jar</packaging> <version>0.0.1-SNAPSHOT</version> <name>Test Spring Hadoop with WebHDFS</name> <description>Test Spring Hadoop with WebHDFS</description> <parent> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-parent</artifactId> <version>1.1.0.RELEASE</version> </parent> <repositories> <repository> <id>spring-milestones</id> <url>http://repo.spring.io/libs-release</url> </repository> </repositories> <properties> <start-class>testwebhdfs.Main</start-class> <java.version>1.6</java.version> <hadoop.version>2.4.1</hadoop.version> </properties> <build> <plugins> <plugin> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-maven-plugin</artifactId> </plugin> </plugins> </build> <dependencies> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter</artifactId> </dependency> <dependency> <groupId>org.springframework.data</groupId> <artifactId>spring-data-hadoop</artifactId> <version>2.0.2.RELEASE</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project>
<?xml version="1.0" encoding="UTF-8"?> <beans xmlns="http://www.springframework.org/schema/beans" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:hdp="http://www.springframework.org/schema/hadoop" xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd http://www.springframework.org/schema/hadoop http://www.springframework.org/schema/hadoop/spring-hadoop.xsd"> <hdp:configuration id="hadoopConfiguration" /> <hdp:file-system uri="webhdfs://localhost:50070" /> </beans>
package testwebhdfs; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.ImportResource; @Configuration @ImportResource("hadoop-context.xml") public class Main implements CommandLineRunner { @Autowired private FileSystem fs; @Override public void run(String... strings) throws Exception { Path[] paths = new Path[strings.length]; for (int i = 0; i < strings.length; ++i) { paths[i] = new Path(strings[i]); } for (FileStatus stat: fs.listStatus(paths)) { System.out.println(stat.getPath()); } } public static void main(String[] args) { SpringApplication.run(Main.class, args); } }