Spring Boot 实现 SFTP 文件上传下载

周三不加班

发布于 2019-06-04 18:42:22

4.7K0

发布于 2019-06-04 18:42:22

1. 实现背景及意义

近期由于系统迁移到docker容器，采用Spring Boot 框架实现微服务治理，在此基础上晚间批量文件服务器也由ftp改成sftp，由于之前ftp的实现是采用公具类的形式，在此基础之上，未屏蔽开发细节和依赖Spring Boot自动装配的特性，进行组件的自动装配和改造，旨在实现简化开发，提高文件传输的安全性和数据交互的可靠性。

2. 什么是SFTP

sftp是SSH File Transfer Protocol的缩写，安全文件传送协议。可以为传输文件提供一种安全的网络的加密方法。sftp 与 ftp 有着几乎一样的语法和功能。SFTP 为 SSH的其中一部分，是一种传输档案至 Blogger 伺服器的安全方式。

其实在SSH软件包中，已经包含了一个叫作SFTP(Secure File Transfer Protocol)的安全文件信息传输子系统，SFTP本身没有单独的守护进程，它必须使用sshd守护进程（端口号默认是22）来完成相应的连接和答复操作，所以从某种意义上来说，SFTP并不像一个服务器程序，而更像是一个客户端程序。

SFTP同样是使用加密传输认证信息和传输的数据，所以，使用SFTP是非常安全的。但是，由于这种传输方式使用了加密/解密技术，所以传输效率比普通的FTP要低得多。

3. SFTP文件传输在java中的实现一

3.1 Maven依赖

<dependency>
 <groupId>com.jcraft</groupId>
 <artifactId>jsch</artifactId>
 <version>0.1.54</version>
</dependency>

3.2 SFTP相关config封装

/**
 * @ClassName: SftpConfig
 * @Description: sftp配置类
 * @Author: 尚先生
 * @CreateDate: 2019/1/7
 * @Version: 1.0
 */
public class SftpConfig {
 private String hostname;
 private Integer port;
 private String username;
 private String password;
 private Integer timeout;
 private Resource privateKey;
 private String remoteRootPath;
 private String fileSuffix;
 public String getHostname() {
 return hostname;
    }
 public void setHostname(String hostname) {
 this.hostname = hostname;
    }
 public Integer getPort() {
 return port;
    }
 public void setPort(Integer port) {
 this.port = port;
    }
 public String getUsername() {
 return username;
    }
 public void setUsername(String username) {
 this.username = username;
    }
 public String getPassword() {
 return password;
    }
 public void setPassword(String password) {
 this.password = password;
    }
 public Integer getTimeout() {
 return timeout;
    }
 public void setTimeout(Integer timeout) {
 this.timeout = timeout;
    }
 public Resource getPrivateKey() {
 return privateKey;
    }
 public void setPrivateKey(Resource privateKey) {
 this.privateKey = privateKey;
    }
 public String getRemoteRootPath() {
 return remoteRootPath;
    }
 public void setRemoteRootPath(String remoteRootPath) {
 this.remoteRootPath = remoteRootPath;
    }
 public String getFileSuffix() {
 return fileSuffix;
    }
 public void setFileSuffix(String fileSuffix) {
 this.fileSuffix = fileSuffix;
    }
 public SftpConfig(String hostname, Integer port, String username, String password, Integer timeout, Resource privateKey, String remoteRootPath, String fileSuffix) {
 this.hostname = hostname;
 this.port = port;
 this.username = username;
 this.password = password;
 this.timeout = timeout;
 this.privateKey = privateKey;
 this.remoteRootPath = remoteRootPath;
 this.fileSuffix = fileSuffix;
    }
 public SftpConfig(String hostname, Integer port, String username, String password, Integer timeout, String remoteRootPath) {
 this.hostname = hostname;
 this.port = port;
 this.username = username;
 this.password = password;
 this.timeout = timeout;
 this.remoteRootPath = remoteRootPath;
    }
 public SftpConfig() {
    }
}

3.3 SFTP工具类实现

/**
 * @ClassName: SFTP
 * @Description: sftp上传通用类
 * @Author: 尚先生
 * @CreateDate: 2019/1/3
 * @Version: 1.0
 */
public class SFTP {
 private long count;
 /**
     * 已经连接次数
     */
 private long count1 = 0;
 private long sleepTime;
 private static final Logger logger = LoggerFactory.getLogger(SFTP.class);
 /**
     * 连接sftp服务器
     *
     * @return
     */
 public ChannelSftp connect(SftpConfig sftpConfig) {
 ChannelSftp sftp = null;
 try {
 JSch jsch = new JSch();
            jsch.getSession(sftpConfig.getUsername(), sftpConfig.getHostname(), sftpConfig.getPort());
 Session sshSession = jsch.getSession(sftpConfig.getUsername(), sftpConfig.getHostname(), sftpConfig.getPort());
            logger.info("Session created ... UserName=" + sftpConfig.getUsername() + ";host=" + sftpConfig.getHostname() + ";port=" + sftpConfig.getPort());
            sshSession.setPassword(sftpConfig.getPassword());
 Properties sshConfig = new Properties();
            sshConfig.put("StrictHostKeyChecking", "no");
            sshSession.setConfig(sshConfig);
            sshSession.connect();
            logger.info("Session connected ...");
            logger.info("Opening Channel ...");
 Channel channel = sshSession.openChannel("sftp");
            channel.connect();
            sftp = (ChannelSftp) channel;
            logger.info("登录成功");
        } catch (Exception e) {
 try {
                count1 += 1;
 if (count == count1) {
 throw new RuntimeException(e);
                }
 Thread.sleep(sleepTime);
                logger.info("重新连接....");
                connect(sftpConfig);
            } catch (InterruptedException e1) {
 throw new RuntimeException(e1);
            }
        }
 return sftp;
    }
 /**
     * 上传文件
     *
     * @param directory  上传的目录
     * @param uploadFile 要上传的文件
     * @param sftpConfig
     */
 public void upload(String directory, String uploadFile, SftpConfig sftpConfig) {
 ChannelSftp sftp = connect(sftpConfig);
 try {
            sftp.cd(directory);
        } catch (SftpException e) {
 try {
                sftp.mkdir(directory);
                sftp.cd(directory);
            } catch (SftpException e1) {
 throw new RuntimeException("ftp创建文件路径失败" + directory);
            }
        }
 File file = new File(uploadFile);
 InputStream inputStream=null;
 try {
            inputStream = new FileInputStream(file);
            sftp.put(inputStream, file.getName());
        } catch (Exception e) {
 throw new RuntimeException("sftp异常" + e);
        } finally {
            disConnect(sftp);
            closeStream(inputStream,null);
        }
    }
 /**
     * 下载文件
     *
     * @param directory    下载目录
     * @param downloadFile 下载的文件
     * @param saveFile     存在本地的路径
     * @param sftpConfig
     */
 public void download(String directory, String downloadFile, String saveFile, SftpConfig sftpConfig) {
 OutputStream output = null;
 try {
 File localDirFile = new File(saveFile);
 // 判断本地目录是否存在，不存在需要新建各级目录
 if (!localDirFile.exists()) {
                localDirFile.mkdirs();
            }
 if (logger.isInfoEnabled()) {
                logger.info("开始获取远程文件:[{}]---->[{}]", new Object[]{directory, saveFile});
            }
 ChannelSftp sftp = connect(sftpConfig);
            sftp.cd(directory);
 if (logger.isInfoEnabled()) {
                logger.info("打开远程文件:[{}]", new Object[]{directory});
            }
            output = new FileOutputStream(new File(saveFile.concat(File.separator).concat(downloadFile)));
            sftp.get(downloadFile, output);
 if (logger.isInfoEnabled()) {
                logger.info("文件下载成功");
            }
            disConnect(sftp);
        } catch (Exception e) {
 if (logger.isInfoEnabled()) {
                logger.info("文件下载出现异常，[{}]", e);
            }
 throw new RuntimeException("文件下载出现异常，[{}]", e);
        } finally {
            closeStream(null,output);
        }
    }
 /**
     * 下载远程文件夹下的所有文件
     *
     * @param remoteFilePath
     * @param localDirPath
     * @throws Exception
     */
 public void getFileDir(String remoteFilePath, String localDirPath, SftpConfig sftpConfig) throws Exception {
 File localDirFile = new File(localDirPath);
 // 判断本地目录是否存在，不存在需要新建各级目录
 if (!localDirFile.exists()) {
            localDirFile.mkdirs();
        }
 if (logger.isInfoEnabled()) {
            logger.info("sftp文件服务器文件夹[{}],下载到本地目录[{}]", new Object[]{remoteFilePath, localDirFile});
        }
 ChannelSftp channelSftp = connect(sftpConfig);
 Vector<LsEntry> lsEntries = channelSftp.ls(remoteFilePath);
 if (logger.isInfoEnabled()) {
            logger.info("远程目录下的文件为[{}]", lsEntries);
        }
 for (LsEntry entry : lsEntries) {
 String fileName = entry.getFilename();
 if (checkFileName(fileName)) {
 continue;
            }
 String remoteFileName = getRemoteFilePath(remoteFilePath, fileName);
            channelSftp.get(remoteFileName, localDirPath);
        }
        disConnect(channelSftp);
    }
 /**
     * 关闭流
     * @param outputStream
     */
 private void closeStream(InputStream inputStream,OutputStream outputStream) {
 if (outputStream != null) {
 try {
                outputStream.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
 if(inputStream != null){
 try {
                inputStream.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
 private boolean checkFileName(String fileName) {
 if (".".equals(fileName) || "..".equals(fileName)) {
 return true;
        }
 return false;
    }
 private String getRemoteFilePath(String remoteFilePath, String fileName) {
 if (remoteFilePath.endsWith("/")) {
 return remoteFilePath.concat(fileName);
        } else {
 return remoteFilePath.concat("/").concat(fileName);
        }
    }
 /**
     * 删除文件
     *
     * @param directory  要删除文件所在目录
     * @param deleteFile 要删除的文件
     * @param sftp
     */
 public void delete(String directory, String deleteFile, ChannelSftp sftp) {
 try {
            sftp.cd(directory);
            sftp.rm(deleteFile);
        } catch (Exception e) {
 throw new RuntimeException(e);
        }
    }
 /**
     * 列出目录下的文件
     *
     * @param directory  要列出的目录
     * @param sftpConfig
     * @return
     * @throws SftpException
     */
 public List<String> listFiles(String directory, SftpConfig sftpConfig) throws SftpException {
 ChannelSftp sftp = connect(sftpConfig);
 List fileNameList = new ArrayList();
 try {
            sftp.cd(directory);
        } catch (SftpException e) {
 return fileNameList;
        }
 Vector vector = sftp.ls(directory);
 for (int i = 0; i < vector.size(); i++) {
 if (vector.get(i) instanceof LsEntry) {
 LsEntry lsEntry = (LsEntry) vector.get(i);
 String fileName = lsEntry.getFilename();
 if (".".equals(fileName) || "..".equals(fileName)) {
 continue;
                }
                fileNameList.add(fileName);
            }
        }
        disConnect(sftp);
 return fileNameList;
    }
 /**
     * 断掉连接
     */
 public void disConnect(ChannelSftp sftp) {
 try {
            sftp.disconnect();
            sftp.getSession().disconnect();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
 public SFTP(long count, long sleepTime) {
 this.count = count;
 this.sleepTime = sleepTime;
    }
 public SFTP() {
    }
}

3.4 测试类实现

/**
 * @ClassName: TestSFTPUtils
 * @Description: SFTP工具类测试类
 * @Author: 尚先生
 * @CreateDate: 2019/4/25 11:09
 * @Version: 1.0
 */
public class TestSFTPUtils {
 private static final Logger logger = LoggerFactory.getLogger(TestSFTPUtils.class);
 public static void main(String[] args) {
            SFTP ftp = new SFTP(3, 6000);
 SftpConfig sftpConfig = new SftpConfig("10.0.155.55", 22, "test", "test", 1000, "/opt/bdepfile/bdp/tset/20190425");
 try {
 List<String> list = ftp.listFiles("/opt/bdepfile/bdp/pucms/20190108", sftpConfig);
                logger.info("文件上传下载详情"  ,new Object[]{list});
            } catch (SftpException e) {
                logger.error("文件上传下载异常:[{}]" ,new Object[]{e});
            }
        }
}

4. SFTP文件传输在java中的实现二

4.1 Maven依赖

<dependency>
 <groupId>com.jcraft</groupId>
 <artifactId>jsch</artifactId>
 <version>0.1.54</version>
</dependency>

4.2 扩展上述工具类的实现

4.2.1 SFTP.java改造

/**
 * @ClassName: SFTP
 * @Description: sftp上传通用类
 * @Author: 尚先生
 * @CreateDate: 2019/1/3
 * @Version: 1.0
 */
@Component
public class SFTP {
 private ChannelSftp channelSftp;
 @Value("${sftp.remotepath}")
 private String remotepath;
 @Value("${sftp.localpath}")
 private String localpath;
 @Value("${sftp.filenames}")
 private String filenames;
 private static final String COMPLATEG_FILE_FLAG = "over_%s.dat";
 private static final Logger logger = LoggerFactory.getLogger(SFTP.class);
    ...
 public void setChannelSftp(ChannelSftp channelSftp) {
 this.channelSftp = channelSftp;
    }
 /**
     *
     * @param remotePath
     * @param remoteFileName
     * @param localPath
     * @param localFileName
     * @return
     */
 public boolean downloadFile(String remotePath, String remoteFileName, String localPath, String localFileName){
        logger.info("开始下载文件，远程路径:[{}],本地路径:[{}],文件名称:[{}]",new Object[]{remotePath,localPath,remoteFileName});
 FileOutputStream fileOutputStream = null;
 File file = new File(localPath + localFileName);
 try {
            fileOutputStream = new FileOutputStream(file);
            channelSftp.get(remotePath + remoteFileName,fileOutputStream);
 return true;
        } catch (Exception e) {
            logger.error("sftp下载文件失败:[{}]",new Object[]{e});
 return false;
        }
    }
 /**
     * 单个ok文件下载
     * @param trandate
     * @return
     */
 public boolean downloadOKFile(String trandate){
        trandate = trandate.replace("-", "");
 String localDirPath = localpath.concat("/").concat(trandate);
 File localDirFile = new File(localDirPath);
 if (!localDirFile.exists()){
            localDirFile.mkdirs();
        }else {
            logger.info("文件[{}]已存在",new Object[]{localDirPath});
 if (!localDirFile.isDirectory()){
                logger.error("文件[{}]已存在，但不是目录，文件下载失败",new Object[]{localDirPath});
 throw new RuntimeException(String.format("本地文件[{%s}]已存在，但不是目录，不能创建文件",localDirPath));
            }
        }
 String filename = String.format(COMPLATEG_FILE_FLAG, trandate);
 String remoteFilePath = remotepath.concat("/").concat(trandate).concat("/");
 String localFilePath = localDirPath.concat("/");
 boolean flag = downloadFile(remoteFilePath, filename, localFilePath, filename);
 return flag;
    }
 /**
     * 多个文件下载
     * @param trandate
     * @return
     */
 public boolean downloadCoreFilesToLocal(String trandate){
 boolean flag = false;
        trandate = trandate.replace("-", "");
 String localDirPath = localpath.concat("/").concat(trandate).concat("/");
 String remoteDirPath = remotepath.concat("/").concat(trandate).concat("/");
 for (String coreFileName : filenames.split(",")){
 //文件名称截取
 String coreFilaName = String.format(coreFileName.trim(), trandate);
            flag = downloadFile(remoteDirPath, coreFileName, localDirPath, coreFileName);
        }
 return flag;
    }
}

4.2.2 自动装配SFTP连接器

/**
 * @ClassName: SftpClientConfigure
 * @Description: 自动装配Sftp连接器
 * @Author: 尚先生
 * @CreateDate: 2019/4/25
 * @Version: 1.0
 */
@Configuration
@ConfigurationProperties(prefix = "sftp")
public class SftpClientConfigure {
 private String hostname;
 private Integer port;
 private String username;
 private String password;
 private Integer timeout;
 private String privateKey;
 private String remoteRootPath;
 private String fileSuffix;
 // 通道类型
 private static final String CHANNEL_TYPE = "sftp";
 private static final Logger logger = LoggerFactory.getLogger(SftpClientConfigure.class);
 public String getHostname() {
 return hostname;
    }
 public void setHostname(String hostname) {
 this.hostname = hostname;
    }
 public Integer getPort() {
 return port;
    }
 public void setPort(Integer port) {
 this.port = port;
    }
 public String getUsername() {
 return username;
    }
 public void setUsername(String username) {
 this.username = username;
    }
 public String getPassword() {
 return password;
    }
 public void setPassword(String password) {
 this.password = password;
    }
 public String getPrivateKey() {
 return privateKey;
    }
 public void setPrivateKey(String privateKey) {
 this.privateKey = privateKey;
    }
 public String getRemoteRootPath() {
 return remoteRootPath;
    }
 public void setRemoteRootPath(String remoteRootPath) {
 this.remoteRootPath = remoteRootPath;
    }
 public String getFileSuffix() {
 return fileSuffix;
    }
 public void setFileSuffix(String fileSuffix) {
 this.fileSuffix = fileSuffix;
    }
 public Integer getTimeout() {
 return timeout;
    }
 public void setTimeout(Integer timeout) {
 this.timeout = timeout;
    }
 @Bean("sshSession")
 @Lazy
 @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
 public Session session() throws JSchException {
 if (logger.isInfoEnabled()) {
            logger.info("获取session，设置的超时时间为[{}]毫秒", timeout);
        }
 JSch jsch = new JSch();
 Session session = jsch.getSession(username, hostname, port);
 // 设置秘钥
//        jsch.addIdentity(privateKey);
        session.setPassword(password);
 Properties config = new Properties();
        config.put("StrictHostKeyChecking", "no");
        session.setConfig(config);  //为Session对象设置properties
        session.setTimeout(timeout);  //设置timeout时间
        session.connect();  //通过Session建立链接
 return session;
    }
 @Bean("coreSftpChannel")
 @Lazy
 @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
 public ChannelSftp channel(Session session) throws JSchException {
 if (logger.isInfoEnabled()) {
            logger.info("初始化sftp连接");
        }
 Channel channel = session.openChannel(CHANNEL_TYPE);  //打开SFTP通道
        channel.connect();  //建立SFTP通道的连接
 return (ChannelSftp) channel;
    }
}

4.2.3 配置文件

# sftp配置sftp.hostname=10.0.155.55
sftp.port=22
sftp.username=test
sftp.password=test
sftp.timeout=6000
sftp.privateKey=
sftp.remotepath=/opt/bdepfile/bdp/tset
sftp.localpath=D:/core
sftp.filenames=duebillInfo_%s.dat,repayInfo_%s.dat

4.2.4 测试类实现

/**
 * @ClassName: TestAutoConfigurationSFTP
 * @Description: SftpClientConfigure测试类
 * @Author: 尚先生
 * @CreateDate: 2019/4/25
 * @Version: 1.0
 */
@RunWith(SpringRunner.class)
@SpringBootTest
public class TestAutoConfigurationSFTP {
 private static final Logger logger = LoggerFactory.getLogger(TestAutoConfigurationSFTP.class);
 @Autowired
 @Qualifier("coreSftpChannel")
 private ChannelSftp channelSftp;
 @Autowired
 private SFTP sftp;
 @Test
 public void testAotuDownload(){
 String trandate = "2019-04-25";
 boolean flag = false;
        sftp.setChannelSftp(channelSftp);
        flag = sftp.downloadOKFile(trandate);
        flag = sftp.downloadCoreFilesToLocal(trandate);
        logger.error("下载文件结果：[{}]",new Object[]{flag});
    }
}

4.2.5 测试结果

打开 Git Bash Here
cd D:\core
ll
 20190425
cd 20190425
ll -als
    ./
    ../
    duebillInfo_20190425.dat
    repayInfo_20190425.dat
    over_20190425.dat

4.3 SFTP扩展实现

由于当前 SpringBoot环境实现，采用的是 SpringBoot"自动装配"实现的，自动注入并调用实现从核心拉取文件的功能。

在项目中取固定文件时，只需动态追加或者替换下面的配置项 sftp.filenames=duebillInfo_%s.dat,repayInfo_%s.dat
如果是新增sftp连接时可以手动创建 SftpConfigconfig=newSftpConfig(); SFTP sftp=newSFTP(); sftp.download(directory,downloadFile,saveFile,sftpConfig)
至此，多种方式实现sftp文件传输可以共存，而且在系统中可以实现"自动装配"。