标签:string 名称 hash dal sre throw int 属性 port
引言:使用多线程下载图片可以大幅度的提升下载速度,基于封装理念把可以抽离出来的代码全部抽离。
代码思路:就是先把需要下载的文件放入一个集合,再下载到一个临时目录,在压缩,最后删除临时目录中的下载文件。
import java.util.HashMap; /** * @author XuYanK * @date 2020/5/18 9:08 */ public class Header { public static final HashMap<String ,String> header = new HashMap<>(); private Header() { } static { header.put("Accept-Encoding", "gzip, deflate"); header.put("Accept-Language", "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2"); header.put("Cache-Control", "max-age=0"); header.put("Connection", "keep-alive"); header.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:59.0) Gecko/20100101 Firefox/59.0"); header.put("Accept", "image/gif,image/jpeg,image/pjpeg,image/pjpeg, " + "application/x-shockwave-flash, application/xaml+xml, " + "application/vnd.ms-xpsdocument, application/x-ms-xbap" + "application/x-ms-application,application/vnd.ms-excel" + "application/vnd.ms-powerpoint, application/msword,*/*"); } }
import java.io.InputStream; import java.io.RandomAccessFile; import java.net.HttpURLConnection; import java.net.URL; import java.util.concurrent.CountDownLatch; /** * @author XuYanK * @date 2020/5/18 9:10 */ public class DownloadThread implements Runnable { //当前线程的下载位置 private int startPos; //定义当前线程负责下载的文件大小 private int currentPartSize; //当前线程需要下载的文件块,此类的实例支持对随机访问文件的读取和写入。 private RandomAccessFile currentPart; //定义该线程已下载的字节数 private int length; //线程计数器 private CountDownLatch latch; private String path; public int getLength() { return length; } public DownloadThread(int startPos,int currentPartSize,RandomAccessFile currentPart, String path ,CountDownLatch latch){ this.startPos = startPos; this.currentPartSize = currentPartSize; this.currentPart = currentPart; this.path = path; this.latch =latch; } @Override public void run(){ InputStream inputStream = null; try{ URL url = new URL(path); HttpURLConnection conn = (HttpURLConnection)url.openConnection(); conn.setConnectTimeout(5 * 1000); //设置请求方法 conn.setRequestMethod("GET"); //设置请求属性 Header.header.forEach((key, value) -> conn.setRequestProperty(key,value)); inputStream = conn.getInputStream(); //inputStream.skip(n);跳过和丢弃此输入流中数据的 n 个字节 inputStream.skip(this.startPos); byte[] buffer = new byte[1024]; int hasRead = 0; //读取网络数据写入本地 while(length < currentPartSize && (hasRead = inputStream.read(buffer)) != -1){ currentPart.write(buffer, 0, hasRead); length += hasRead; } } catch(Exception e){ e.printStackTrace(); }finally { try { currentPart.close(); inputStream.close(); }catch (Exception e){ e.printStackTrace(); } latch.countDown(); } } }
import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.RandomAccessFile; import java.math.BigDecimal; import java.net.HttpURLConnection; import java.net.URL; import java.util.concurrent.CountDownLatch; /** * @author XuYanK * @date 2020/5/18 9:10 */ public class DownUtil { private Logger logger = LoggerFactory.getLogger(DownUtil.class); //定义下载路径 private String path; //指定所下载的文件的保存位置 private String targetFile; //定义下载线程的数量 private int threadNum; //定义下载线程的对象 private DownloadThread[] threads; //下载文件的总大小 private int fileSize; //线程计数器 private CountDownLatch latch; public DownUtil(String path, String targetFile, int threadNum){ this.path = path; this.targetFile = targetFile; this.threadNum = threadNum; threads = new DownloadThread[threadNum]; latch = new CountDownLatch(threadNum); } /** * 多线程下载文件 * @throws Exception */ public void downLoad() throws Exception{ long t1 = System.currentTimeMillis(); URL url = new URL(path); HttpURLConnection conn = (HttpURLConnection)url.openConnection(); conn.setConnectTimeout(5 * 1000); //设置请求方法 // conn.setRequestMethod("POST"); //设置请求属性 Header.header.forEach((key, value) -> conn.setRequestProperty(key,value)); //得到文件大小 fileSize = conn.getContentLength(); conn.disconnect(); int currentPartSize = fileSize / threadNum + 1; RandomAccessFile file = new RandomAccessFile(targetFile,"rw"); //设置本地文件大小 file.setLength(fileSize); file.close(); for(int i = 0;i < threadNum;i++){ //计算每个线程的下载位置 int startPos = i * currentPartSize; //每个线程使用一个RandomAccessFile进行下载 RandomAccessFile currentPart = new RandomAccessFile(targetFile,"rw"); //定位该线程的下载位置 currentPart.seek(startPos); //创建下载线程 threads[i] = new DownloadThread(startPos, currentPartSize, currentPart , path , latch); Thread t = new Thread(threads[i]); t.start(); } BigDecimal n = BigDecimal.ZERO; while (true){ BigDecimal length = BigDecimal.ZERO; if (latch.getCount()==0 && BigDecimal.ONE.compareTo(n)<=0){break;} for (DownloadThread downloadThread:threads) { length = length.add(new BigDecimal(downloadThread.getLength())); } n = length.divide(new BigDecimal(fileSize), 4, BigDecimal.ROUND_HALF_UP); logger.info("已下载"+n.multiply(new BigDecimal("100"))+"%"); Thread.sleep(1000); } long t2 = System.currentTimeMillis(); logger.info("下载完成,文件大小为"+fileSize*1.0/(1024*1024)+"M,共用时"+(t2-t1)*1.0/1000+"秒"); //线程等待,一个文件下载成功后继续下一个 latch.await(); } /** * 根据文件名获取文件类型 * @param fileName * @return */ public static String getFileType(String fileName) { return fileName.substring(fileName.lastIndexOf(".")); } }
import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; import java.util.ArrayList; import java.util.List; import java.util.zip.CRC32; import java.util.zip.CheckedOutputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; /** * @author XuYanK * @date 2020/5/18 9:45 */ public class ZipUtils { private static Logger logger = LoggerFactory.getLogger(ZipUtils.class); /** 缓冲器大小 */ private static final int BUFFER = 512; /**压缩得到的文件的后缀名*/ private static final String SUFFIX=".zip"; /** *文件写入压缩包 * @param sb * @param zipname 压缩包名称 * @throws IOException */ public static void writeZip(StringBuffer sb, String zipname) throws IOException { String[] files = sb.toString().split(","); OutputStream os = new BufferedOutputStream(new FileOutputStream(zipname + SUFFIX)); ZipOutputStream zos = new ZipOutputStream(os); byte[] buf = new byte[8192]; int len; for (int i = 0; i < files.length; i++) { File file = new File(files[i]); if (!file.isFile()) { continue; } ZipEntry ze = new ZipEntry(file.getName()); zos.putNextEntry(ze); BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file)); while ((len = bis.read(buf)) > 0) { zos.write(buf, 0, len); } zos.closeEntry(); } zos.closeEntry(); zos.close(); } /** * 得到源文件路径的所有文件 * @param dirFile 压缩源文件路径 * */ public static List<File> getAllFile(File dirFile){ List<File> fileList=new ArrayList<>(); File[] files= dirFile.listFiles(); //文件 for(File file:files){ if(file.isFile()){ fileList.add(file); logger.info("文件添加到压缩列表:"+file.getName()); }else {//目录 //非空目录 if(file.listFiles().length!=0){ //把递归文件加到fileList中 fileList.addAll(getAllFile(file)); }else {//空目录 fileList.add(file); logger.info("空目录添加到压缩列表:"+file.getName()); } } } return fileList; } /** * 压缩文件 * @param dirPath 压缩源文件路径 * @param zipFileName 压缩目标文件路径 * */ public static void compress(String dirPath,String zipFileName) { //添加文件的后缀名 zipFileName = zipFileName + SUFFIX; File dirFile = new File(dirPath); List<File> fileList = getAllFile(dirFile); byte[] buffer = new byte[BUFFER]; ZipEntry zipEntry = null; //每次读取出来的长度 int readLength = 0; try { // 对输出文件做CRC32校验 CheckedOutputStream cos = new CheckedOutputStream(new FileOutputStream( zipFileName), new CRC32()); ZipOutputStream zos = new ZipOutputStream(cos); for (File file : fileList) { //若是文件,则压缩文件 if (file.isFile()) { zipEntry = new ZipEntry(getRelativePath(dirPath, file)); zipEntry.setSize(file.length()); zipEntry.setTime(file.lastModified()); zos.putNextEntry(zipEntry); InputStream is = new BufferedInputStream(new FileInputStream(file)); while ((readLength = is.read(buffer, 0, BUFFER)) != -1) { zos.write(buffer, 0, readLength); } is.close(); logger.info("file compress:" + file.getCanonicalPath()); System.out.println(); } else { //若是空目录,则写入zip条目中 zipEntry = new ZipEntry(getRelativePath(dirPath, file)); zos.putNextEntry(zipEntry); logger.info("dir compress: " + file.getCanonicalPath() + "/"); } } zos.close(); //最后得关闭流,不然压缩最后一个文件会出错 } catch (FileNotFoundException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } } /** * 获取相对路径 * @param dirPath 源文件路径 * @param file 准备压缩的单个文件 * */ public static String getRelativePath(String dirPath,File file){ File dirFile=new File(dirPath); String relativePath=file.getName(); while (true){ file=file.getParentFile(); if(file==null) break; if(file.equals(dirFile)){ break; } else { relativePath=file.getName()+"/"+relativePath; } } return relativePath; } /** * 删除文件夹目录下的文件 * @param path * @return */ public static boolean deleteDir(String path) { File file = new File(path); //判断是否待删除目录是否存在 if (!file.exists()) { logger.warn("文件目录不存在"); return false; } //取得当前目录下所有文件和文件夹 String[] content = file.list(); for (String name : content) { File temp = new File(path, name); //判断是否是目录 if (temp.isDirectory()) { //递归调用,删除目录里的内容 deleteDir(temp.getAbsolutePath()); //删除空目录 temp.delete(); } else { //直接删除文件 if (!temp.delete()) { logger.warn("删不掉,使用强制删除方法GC()"); //强制删除 // System.gc(); // temp.delete(); } } } return true; } }
import com.alibaba.fastjson.JSONObject; import lombok.extern.slf4j.Slf4j; import java.io.File; import java.util.List; import java.util.Map; /** * @author XuYanK * @date 2020/5/18 9:13 */ @Slf4j public class MyDowloadTest { public static void main(String[] args) { //系统本地临时文件夹路径 String targetFlie =System.getProperty("java.io.tmpdir")+File.separator+"testDown"; //判断是否存在该文件夹 File file = new File(targetFlie); if (!file.exists()) { file.mkdirs(); } String listData = "[{\n" + "\t\t\"userId\": \"1\",\n" + "\t\t\"username\": \"admin\",\n" + "\t\t\"password\": \"123456\",\n" + "\t\t\"imgpath\": \"http://c.hiphotos.baidu.com/zhidao/pic/item/54fbb2fb43166d22d552b941432309f79052d23a.jpg\"\n" + "\t},\n" + "\t{\n" + "\t\t\"userId\": \"2\",\n" + "\t\t\"username\": \"jacks\",\n" + "\t\t\"password\": \"111111\",\n" + "\t\t\"imgpath\": \"http://e.hiphotos.baidu.com/zhidao/pic/item/b8014a90f603738d8d7adbbbb31bb051f819ec73.jpg\"\n" + "\t}\n" + "]"; //转换list对象 List<Map> lists = JSONObject.parseArray(listData, Map.class); for (int i = 0; i < lists.size(); i++) { String imgpath = (String) lists.get(i).get("imgpath"); //获取文件后缀 String fileType = DownUtil.getFileType(imgpath); //多线程下载列表图片 DownUtil downUtil = new DownUtil( imgpath, targetFlie + "\\" + System.currentTimeMillis() + fileType, 8); try { downUtil.downLoad(); } catch (Exception e) { e.printStackTrace(); } } String zipName = "测试"; ZipUtils.compress(targetFlie, zipName); ZipUtils.deleteDir(targetFlie); // FileUtils.deleteQuietly(new File(targetFlie)); } }
标签:string 名称 hash dal sre throw int 属性 port
原文地址:https://www.cnblogs.com/21-Gram/p/12930965.html