1. #!/bin/env python
    2. # -*- coding: utf-8 -*-
    4. import datetime
    5. import os
    6. import fileinput
    7. import psycopg2
    8. import tarfile
    9. import shutil
    10. import logging
    11. import time
    13. logging.basicConfig(level=logging.DEBUG,
    14.                     format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',   # 日志格式
    15.                     datefmt='%Y-%m-%d %H:%M:%S',    # 时间格式：2018-11-12 23:50:21
    16.                     filename='/home/hjpt/App/hplatform/logs/hplatform_local_scan_del_lsms.log',    # 日志的输出路径
    17.                     filemode='a')
    20. #获取目录下的所有文件
    21. def scan_file(infor):
    22.     tar_file = []
    23.     remote_path = infor[1]
    24.     for files in os.listdir(remote_path):
    25.         print files
    26.         file_name = remote_path + '/' + files
    27.         print file_name
    28.         if os.path.splitext(file_name)[1] == '.gz':
    29.             tar_file.append(files)
    30.     print tar_file
    31.     return tar_file
    34. #删除文件的第一行
    35. def del_file_first(del_file):
    36.     for line in fileinput.input(del_file, inplace=1):
    37.         if not fileinput.isfirstline():
    38.             print line.replace("\n","")
    41. #解压缩模块
    42. def un_tar(file_names):
    43.     un_tar_file = []
    44.     ufile = tarfile.open(file_names)
    45.     names = ufile.getnames()
    46.     for tt in names:
    47.         ufile.extract(tt, path=remote_path)
    48.         un_tar_file.append(tt)
    49.         print un_tar_file
    50.         ufile.close()
    51.     return un_tar_file
    55. if __name__ == '__main__':
    57.     sql = '''
    58.         SELECT
    59.         scan_group_id,
    60.         remote_path,
    61.         remote_bk_path,
    62.         local_path,
    63.         file_regular,
    64.         chkf_regular,
    65.         mv_flag,
    66.         data_source,
    67.         table_attr,
    68.         prov_id
    69.         FROM
    70.         public.conf_path_info
    71.         WHERE
    72.         is_valid = '0' and scan_group_id = 'XXXX'
    73.     '''
    75.     try:
    76.         pg_conn = psycopg2.connect(database='XX', user='XX', password='XXXX', host='XXXX', port='XXXX')
    77.         cur = pg_conn.cursor()
    78.         cur.execute(sql)
    79.         rows = cur.fetchall()
    80.         print(rows)
    81.         pg_conn.close()
    82.     except Exception as e:
    83.         print e
    84.     else:
    85.         while True:
    86.             for i in rows:
    87.                 remote_path = i[1]
    88.                 remote_bk_path = i[2]
    89.                 local_path = i[3]
    90.                 files = scan_file(i)
    91.                 #获取压缩文件名称
    92.                 for filename in files:
    93.                     print filename
    94.                     file = remote_path + '/' + filename
    95.                     bak_file = remote_bk_path + '/' + filename
    96.                     try:
    97.                         if os.path.getsize(file) == 0:
    98.                             os.remove(file)
    99.                             logging.info(file + 'file size is : 0 ,delete from local.')
    100.                         else:
    101.                             sss = os.path.getsize(file)
    102.                             #解压压缩文件
    103.                             un_tar_file=un_tar(file)
    104.                             logging.info(file + 'file size is :' + str(sss) + ': untar file successful !')
    105.                             #将文件从正式目录挪到bak目录
    106.                             shutil.move(file,bak_file)
    107.                             for unfilename in un_tar_file:
    108.                                 dfilename = remote_path + '/' + unfilename
    109.                                 tmpfilename = local_path + '/' + unfilename
    110.                                 #删除文件的第一行
    111.                                 del_file_first(dfilename)
    112.                                 logging.info(dfilename + 'delete first line successful !')
    113.                                 shutil.move(dfilename,tmpfilename)
    114.                                 del un_tar_file[0]
    115.                     except Exception as e:
    116.                         print e