多线程分块(按行进行)读取多个文件

"""
多线程分块(按行进行)读取多个文件
1、统计文件的行数
2、根据线程数对文件行数进行分块
3、多线程根据分块行进行读取文件
"""
import os
from concurrent.futures.thread import ThreadPoolExecutor
import time


def wc_file(file):
    """
    统计文件行数
    :return:
    """
    count = 0
    for index, line in enumerate(open(file, "r", encoding="gbk", errors="ignore")):
        count += 1
    return count


def part_wc_file(file, thread_num):
    """
    根据统计的行数对文件按行进行分块
    :return:
    """
    rows_count = wc_file(file)
    num = rows_count // thread_num  # 计算每块的行数
    i = 1
    tt = []  # 存放分块的行数
    t = 0
    while i <= thread_num - 1:
        tt.append((t + 1, i * num))
        t = i * num
        i += 1
    tt.append((t + 1, rows_count))
    return tt


def Read_file(args):
    """
    分块读取文件
    :param tt:
    :param file:
    :return:
    """
    # print(args[0][0], args[0][1])
    count = 0
    with open(args[1], "r", encoding="gbk", errors="ignore") as f:
        for line in f:
            count += 1
            if args[0][0] <= count <= args[0][1]:
                pass
                # print(line)
                # break
    f.close()


def thread_pool_readfile(file):
    """
    多线程分块读取文件
    :return:
    """
    thread_num = 4
    tt = part_wc_file(file, thread_num=thread_num)
    p = ThreadPoolExecutor(thread_num)
    for i in range(len(tt)):
        p.submit(Read_file, args=(tt[i], file))
        # time.sleep(2)
    p.submit(True)


if __name__ == '__main__':
    file = r"/data/gatherfiles/datafiles/047_user_tree/20190804/data.txt"
    start = time.clock()
    thread_pool_readfile(file)
    end = time.clock()
    print("Cost time  %s seconds" % (end - start))