多线程大量读取文件 |
您所在的位置:网站首页 › python多线程读取文件夹下的文件 › 多线程大量读取文件 |
新公司入职一个多月了,至今没有事情可以做,十来个新同事都一样抓狂,所以大家都自己学习一些新东西,我最近在看zookeeper,感觉蛮不错的,和微服务的zuul以及eureka功能类似,只是代码复杂了一些。而今天,我所要说的是java多线程读取文件的两个例子; 例子1:java多线程批量读取文件 package face.thread.ReadFile; /** * 多线程读、写文件 * */ import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; import java.io.FileNotFoundException; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; public class CompareTest3 { public static void main(String args[]) { long millis1 = System.currentTimeMillis(); System.out.println(millis1); Read3 read = new Read3(millis1); ExecutorService service = Executors.newFixedThreadPool(5); for(int i = 1; i Object o = new Object(); List filePathsList = new ArrayList(); int index = 0; private long millis ; public Read3(long millis1 ) { this.millis = millis1; File f = new File("d:" + File.separator + "gc2"); getFileList(f); } private void getFileList(File f) { File[] filePaths = f.listFiles(); for (File s : filePaths) { if (s.isDirectory()) { getFileList(s); } else { if (-1 != s.getName().lastIndexOf(".txt")) { filePathsList.add(s); } } } } public void run() { File file = null; File f2 = null; while (index < filePathsList.size()) { //此处,保证了多线程不会交叉读取文件 //--1.1方法内的变量是线程安全的 //解释:由于方法内的变量是私有的,本体访问的同时别人访问不了,所以是线程安全的。 //--1.2实例变量是非线程安全的 //解释:由于实例变量可以由多个线程访问,当本体操作变量过程中,别人也可以抢占资源操作变量,使数据不同步了,所以是非线程安全的。 synchronized (o) { if (index > filePathsList.size()) { return; } file = filePathsList.get(index); index++; //System.out.println("内部index: " + index); }// System.out.println("文件: " + file.getName()); FileReader fr = null; BufferedReader br = null; StringBuffer sb = new StringBuffer(); FileWriter fw = null; BufferedWriter bw = null; f2 = new File("d:" + File.separator + "gc3" + File.separator + file.getName()); try { fr = new FileReader(file); br = new BufferedReader(fr); fw = new FileWriter(f2); bw = new BufferedWriter(fw); String data = ""; while((data = br.readLine()) != null){ // sb.append(data + "\r"); bw.write(data + "\r"); } bw.write("---------------" + Thread.currentThread().getName()+"---------------"); System.out.println(Thread.currentThread().getName() + " : " + file.getName()); } catch (FileNotFoundException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); }finally{ try { bw.close(); br.close(); /*long millis2 = System.currentTimeMillis(); System.out.println(millis2); System.out.println(millis2 - millis); //大约1-2ms*/ } catch (IOException e) { e.printStackTrace(); } } } } }
例子2:同样的读取文件,改为单线程读取 package face.thread.ReadFile; /** * 单线程读、写文件 */ import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; import java.io.FileNotFoundException; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; public class CompareTest4 { public static void main(String args[]) { final long millis1 = System.currentTimeMillis(); final CyclicBarrier cb = new CyclicBarrier(1,new Runnable(){ public void run() { long millis2 = System.currentTimeMillis(); System.out.println(millis2); System.out.println(millis2 - millis1); //大约1-2ms } }); Read4 read = new Read4(cb); ExecutorService service = Executors.newFixedThreadPool(1); for(int i = 1; i Object o = new Object(); List filePathsList = new ArrayList(); int index = 0; CyclicBarrier cb2; public Read4(CyclicBarrier cb) { this.cb2 = cb; File f = new File("d:" + File.separator + "gc2"); getFileList(f); } private void getFileList(File f) { File[] filePaths = f.listFiles(); for (File s : filePaths) { if (s.isDirectory()) { getFileList(s); } else { if (-1 != s.getName().lastIndexOf(".txt")) { filePathsList.add(s); } } } } public void run() { File file = null; File f2 = null; while (index < filePathsList.size()) { synchronized (o) { if (index > filePathsList.size()) { return; } file = filePathsList.get(index); index++; //System.out.println("内部index: " + index); } // System.out.println("文件: " + file.getName()); FileReader fr = null; BufferedReader br = null; StringBuffer sb = new StringBuffer(); FileWriter fw = null; BufferedWriter bw = null; f2 = new File("d:" + File.separator + "gc3" + File.separator + file.getName()); try { fr = new FileReader(file); br = new BufferedReader(fr); fw = new FileWriter(f2); bw = new BufferedWriter(fw); String data = ""; while((data = br.readLine()) != null){ // sb.append(data + "\r"); bw.write(data + "\r"); } bw.write("---------------" + Thread.currentThread().getName()+"---------------"); } catch (FileNotFoundException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); }finally{ try { bw.close(); br.close(); try { cb2.await(); } catch (InterruptedException e) { e.printStackTrace(); } catch (BrokenBarrierException e) { // TODO Auto-generated catch block e.printStackTrace(); } } catch (IOException e) { e.printStackTrace(); } } } } }
两个例子中,打印的时间即代表线程读取每个代码的时间,性能对比一看就能体现出来,只是个小Demo,望大神勿喷! |
今日新闻 |
推荐新闻 |
CopyRight 2018-2019 办公设备维修网 版权所有 豫ICP备15022753号-3 |