多线程大量读取文件

您所在的位置:网站首页 python多线程读取文件夹下的文件 多线程大量读取文件

多线程大量读取文件

#多线程大量读取文件| 来源: 网络整理| 查看: 265

新公司入职一个多月了,至今没有事情可以做,十来个新同事都一样抓狂,所以大家都自己学习一些新东西,我最近在看zookeeper,感觉蛮不错的,和微服务的zuul以及eureka功能类似,只是代码复杂了一些。而今天,我所要说的是java多线程读取文件的两个例子;

例子1:java多线程批量读取文件

package face.thread.ReadFile;

/**  * 多线程读、写文件  *  */ import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; import java.io.FileNotFoundException; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors;

public class CompareTest3 {     public static void main(String args[]) {      long millis1 = System.currentTimeMillis();      System.out.println(millis1);      Read3 read = new Read3(millis1);      ExecutorService service = Executors.newFixedThreadPool(5);      for(int i = 1; i  Object o = new Object();     List filePathsList = new ArrayList();     int index = 0;     private long millis ;     public Read3(long millis1 ) {      this.millis = millis1;         File f = new File("d:" + File.separator + "gc2");         getFileList(f);     }       private void getFileList(File f) {         File[] filePaths = f.listFiles();         for (File s : filePaths) {             if (s.isDirectory()) {                 getFileList(s);             } else {                 if (-1 != s.getName().lastIndexOf(".txt")) {                     filePathsList.add(s);                 }             }         }     }       public void run() {         File file = null;         File f2 = null;         while (index < filePathsList.size()) {          //此处,保证了多线程不会交叉读取文件

//--1.1方法内的变量是线程安全的 //解释:由于方法内的变量是私有的,本体访问的同时别人访问不了,所以是线程安全的。 //--1.2实例变量是非线程安全的 //解释:由于实例变量可以由多个线程访问,当本体操作变量过程中,别人也可以抢占资源操作变量,使数据不同步了,所以是非线程安全的。             synchronized (o) {                 if (index > filePathsList.size()) {                     return;                 }                 file = filePathsList.get(index);                 index++;                 //System.out.println("内部index: " + index);             }

        //    System.out.println("文件: " + file.getName());             FileReader fr = null;             BufferedReader br = null;             StringBuffer sb = new StringBuffer();                         FileWriter fw  = null;             BufferedWriter bw = null;             f2 = new File("d:" + File.separator + "gc3" + File.separator + file.getName());                try {     fr = new FileReader(file);        br = new BufferedReader(fr);               fw = new FileWriter(f2);     bw = new BufferedWriter(fw);             String data = "";     while((data = br.readLine()) != null){     // sb.append(data + "\r");      bw.write(data + "\r");     }          bw.write("---------------" + Thread.currentThread().getName()+"---------------");     System.out.println(Thread.currentThread().getName() + " : " + file.getName());    } catch (FileNotFoundException e) {     e.printStackTrace();    } catch (IOException e) {     e.printStackTrace();    }finally{     try {      bw.close();      br.close();            /*long millis2 = System.currentTimeMillis();         System.out.println(millis2);            System.out.println(millis2 - millis);  //大约1-2ms*/     } catch (IOException e) {      e.printStackTrace();     }    }         }     } }

例子2:同样的读取文件,改为单线程读取

package face.thread.ReadFile;

/**  * 单线程读、写文件  */ import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; import java.io.FileNotFoundException; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors;

public class CompareTest4 {     public static void main(String args[]) {            final long millis1 = System.currentTimeMillis();      final CyclicBarrier cb = new CyclicBarrier(1,new Runnable(){             public void run() {                 long millis2 = System.currentTimeMillis();               System.out.println(millis2);                 System.out.println(millis2 - millis1);  //大约1-2ms             }         });                    Read4 read = new Read4(cb);      ExecutorService service = Executors.newFixedThreadPool(1);      for(int i = 1; i    Object o = new Object();     List filePathsList = new ArrayList();     int index = 0;     CyclicBarrier cb2;     public Read4(CyclicBarrier cb) {      this.cb2 = cb;         File f = new File("d:" + File.separator + "gc2");         getFileList(f);     }       private void getFileList(File f) {         File[] filePaths = f.listFiles();         for (File s : filePaths) {             if (s.isDirectory()) {                 getFileList(s);             } else {                 if (-1 != s.getName().lastIndexOf(".txt")) {                     filePathsList.add(s);                 }             }         }     }       public void run() {         File file = null;         File f2 = null;         while (index < filePathsList.size()) {             synchronized (o) {                 if (index > filePathsList.size()) {                     return;                 }                 file = filePathsList.get(index);                 index++;                 //System.out.println("内部index: " + index);             }

        //    System.out.println("文件: " + file.getName());             FileReader fr = null;             BufferedReader br = null;             StringBuffer sb = new StringBuffer();                         FileWriter fw  = null;             BufferedWriter bw = null;             f2 = new File("d:" + File.separator + "gc3" + File.separator + file.getName());                try {     fr = new FileReader(file);        br = new BufferedReader(fr);               fw = new FileWriter(f2);     bw = new BufferedWriter(fw);             String data = "";     while((data = br.readLine()) != null){     // sb.append(data + "\r");      bw.write(data + "\r");     }          bw.write("---------------" + Thread.currentThread().getName()+"---------------");    } catch (FileNotFoundException e) {     e.printStackTrace();    } catch (IOException e) {     e.printStackTrace();    }finally{     try {      bw.close();      br.close();       try {       cb2.await();      } catch (InterruptedException e) {       e.printStackTrace();      } catch (BrokenBarrierException e) {       // TODO Auto-generated catch block       e.printStackTrace();      }     } catch (IOException e) {      e.printStackTrace();     }    }         }     } }

两个例子中,打印的时间即代表线程读取每个代码的时间,性能对比一看就能体现出来,只是个小Demo,望大神勿喷!




【本文地址】


今日新闻


推荐新闻


CopyRight 2018-2019 办公设备维修网 版权所有 豫ICP备15022753号-3