腾讯云COS上传、批量删除工具(Python)

  • A+
所属分类:Python

腾讯云对象存储COS是类似于阿里云OSS,相比OSS,COS提供每月免费额度:存储空间50G、外网访问流量10G(内网免费)、免费读请求100万次、写请求10万次。对网站备份来说不错,但是,腾讯云提供的工具太low,参考阿里云OSS,写了一个cos信息配置、创建目录、上传、批量删除工具(coscmd)

安装Python2.7

此工具在Python2.7上测试通过,建议用《OneinStack》安装Python2.7,安装路径为:/usr/local/python,命令如下:

  1. cd ~/oneinstack
  2. ./addons.sh #选择7,install Let's Encrypt client

配置COS

登陆腾讯云管理后台https://console.qcloud.com/cos/bucket创建bucket,并获取API密钥,在下面配置中一一对应。

  1. cd oneinstack/tools #必须进入该目录执行
  2. wget http://mirrors.linuxeye.com/oneinstack/tools/cos.tgz #v4版本
  3. wget http://mirrors.linuxeye.com/oneinstack/tools/cos_v3.tgz #v3版本,可提交工单升v4
  4. tar xzf cos.tgz
  5. /usr/local/python/bin/python ./coscmd config --appid=[appid] --id=[secret_id] --key=[secret_key] --region=[region] --bucket=[bucket]  #v4配置
  6. /usr/local/python/bin/python ./coscmd config --appid=[appid] --id=[secret_id] --key=[secret_key] --bucket=[bucket]  #v3配置

COSv4版本配置

COSv3版本配置

执行后会将相关信息写到~/.coscredentials,执行其它动作会自动加在改配置。

创建目录

上传文件

批量删除

coscmd代码(COSv4)

  1. #!/usr/bin/env python
  2. #coding:utf-8
  3. import sys,os
  4. import datetime
  5. import random
  6. import threading
  7. import time
  8. import datetime
  9. import logging
  10. import ConfigParser
  11. from optparse import OptionParser
  12. from logging.handlers import RotatingFileHandler
  13. from time import strftime, localtime
  14. from time import sleep
  15. from datetime import date
  16. from datetime import timedelta
  17. from cos import CosClient
  18. from cos import UploadFileRequest
  19. from cos import CreateFolderRequest
  20. from cos import DelFileRequest
  21. from cos import DelFolderRequest
  22. from cos import ListFolderRequest
  23. from cos import threadpool
  24. MAX_RETRY_TIMES = 3
  25. LOG_SAVE_EVERY_NUM = 1024
  26. ONE_TASK_DEL_FILE_NUMS = 50
  27. log_level = 1
  28. log_file_name = "del_file.log"
  29. dir_thread_num = 2
  30. file_thread_num = 5
  31. log_out_to_screen = 1
  32. delete_folder_fail_exist = 0
  33. CONFIGFILE = "%s/.coscredentials" % os.path.expanduser('~')
  34. CONFIGSECTION = 'COSCredentials'
  35. HAS_FORK = hasattr(os, 'fork')
  36. HELP = \
  37. '''coscmd:
  38.     config         --appid=[appid] --id=[secret_id] --key=[secret_key] --region=[region] --bucket=[bucket] 
  39.     ls             cosdir
  40.     mkdir          dirname
  41.     put            localfile  cosdir 
  42.     rm(delete,del) object
  43.     '''
  44. CMD_LIST = {}
  45. def cmd_configure(args, options):
  46.     if options.appid is None or options.secret_id is None or options.secret_key is None  or options.region is None or options.bucket is None:
  47.         print("%s miss parameters, use --appid=[appid] --id=[secret_id] --key=[secret_key] --region=[region] --bucket=[bucket] to specify appid/id/key/region/bucket pair" % args[0])
  48.         sys.exit(-1)
  49.     config = ConfigParser.RawConfigParser()
  50.     config.add_section(CONFIGSECTION)
  51.     config.set(CONFIGSECTION, 'appid', options.appid)
  52.     config.set(CONFIGSECTION, 'secret_id', options.secret_id)
  53.     config.set(CONFIGSECTION, 'secret_key', options.secret_key)
  54.     if options.region in ['sh','gz','tj','sgp']:
  55.         config.set(CONFIGSECTION, 'region', options.region)
  56.     else:
  57.         print("input region error, setup use : --region={sh,gz,tj,sgp}")
  58.         sys.exit(-1)
  59.     config.set(CONFIGSECTION, 'bucket', options.bucket)
  60.     cfgfile = open(CONFIGFILE, 'w+')
  61.     config.write(cfgfile)
  62.     print("Your configuration is saved into %s ." % CONFIGFILE)
  63.     cfgfile.close()
  64.     import stat
  65.     os.chmod(CONFIGFILE, stat.S_IREAD | stat.S_IWRITE)
  66. def cmd_loadconfigure():
  67.     config = ConfigParser.ConfigParser()
  68.     config.read(CONFIGFILE)
  69.     global appid
  70.     global secret_id
  71.     global secret_key
  72.     global region
  73.     global bucket
  74.     appid = int(config.get(CONFIGSECTION, 'appid'))
  75.     secret_id = config.get(CONFIGSECTION, 'secret_id').decode('utf-8')
  76.     secret_key = config.get(CONFIGSECTION, 'secret_key').decode('utf-8')
  77.     region = config.get(CONFIGSECTION, 'region')
  78.     bucket = config.get(CONFIGSECTION, 'bucket').decode('utf-8')
  79.     if len(secret_id) == 0 or len(secret_key) == 0 or len(region) == 0 or len(bucket) == 0:
  80.         print("can't get appid/secret_id/secret_key/region/bucket, setup use : config --appid=[appid] --id=[secret_id] --key=[secret_key] --region=[region] --bucket=[bucket]")
  81.         sys.exit(1)
  82. def cmd_lsdir(COSDIR):
  83.     cosdir = COSDIR.decode('utf-8')
  84.     request = ListFolderRequest(bucket, cosdir)
  85.     list_folder_ret = cos_client.list_folder(request)
  86.     if list_folder_ret[u'code'] == 0:
  87.         print(True)
  88.     else:
  89.         print("%s, appid/secret_id/secret_key/region/bucket invalid"% list_folder_ret[u'message'])
  90. def cmd_mkdir(COSDIR):
  91.     cosdir = COSDIR.decode('utf-8')
  92.     request = CreateFolderRequest(bucket, cosdir)
  93.     create_folder_ret = cos_client.create_folder(request)
  94.     if create_folder_ret[u'code'] == 0:
  95.         print("mkdir cos://%s%s OK" % (bucket,COSDIR))
  96.     else:
  97.         print(create_folder_ret[u'message'])
  98. def cmd_put(LOCALFILE,COSFILE):
  99.     localfile = LOCALFILE.decode('utf-8')
  100.     cosfile = COSFILE.decode('utf-8')
  101.     request = UploadFileRequest(bucket, cosfile, localfile)
  102.     request.set_insert_only(0)
  103.     upload_file_ret = cos_client.upload_file(request)
  104.     if upload_file_ret[u'code'] == 0:
  105.         print("put cos://%s%s OK" % (bucket,COSFILE))
  106.     else:
  107.         print(upload_file_ret[u'message'])
  108. def loginit():
  109.     global config
  110.     if (log_file_name == ""):
  111.         return
  112.     log_level = logging.ERROR
  113.     if log_level == 0:
  114.         log_level = logging.DEBUG
  115.     if log_level == 1:
  116.         log_level = logging.INFO
  117.     if log_level == 2:
  118.         log_level = logging.WARNING
  119.         #定义一个RotatingFileHandler,最多备份5个日志文件,每个日志文件最大20M
  120.     logger = logging.getLogger("")
  121.     Rthandler = RotatingFileHandler(log_file_name, maxBytes= 20*1024*1024,backupCount=5)
  122.     Rthandler.setLevel(log_level)
  123.     formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
  124.     Rthandler.setFormatter(formatter)
  125.     logger.addHandler(Rthandler)
  126.         #输出日志到屏幕
  127.     console = logging.StreamHandler()
  128.     console.setFormatter(formatter)
  129.     if (log_out_to_screen == 1):
  130.         logger.addHandler(console)
  131.     logger.setLevel(log_level)
  132.     return logger
  133. #日期相关操作
  134. class Dateop():
  135.     @staticmethod
  136.     def isValidDate(str):
  137.         try:
  138.             time.strptime(str, "%Y""%m""%d")
  139.             return True
  140.         except:
  141.             return False
  142.     @staticmethod
  143.     def getdaystr(n=0):
  144.         dt = date.today()-timedelta(days=n)
  145.         tt = dt.timetuple()
  146.         daystr = strftime("%Y""%m""%d",tt)
  147.         return daystr
  148.     @staticmethod
  149.     def cmpDateAgo(t1,t2):
  150.         if (Dateop.isValidDate(t1)==False or Dateop.isValidDate(t2)==False):
  151.             return False
  152.         if (int(t1) <= int (t2)):
  153.             return True
  154.         return False
  155.     @staticmethod
  156.     def isNeedDeleteDir(dirname, n=0):
  157.         if (len(dirname) != 8):
  158.             return False
  159.         if Dateop.isValidDate(dirname) == False:
  160.             return False
  161.         d2 = Dateop.getdaystr(n);
  162.         if Dateop.cmpDateAgo(dirname, d2):
  163.             return True
  164.         return False
  165. #删除文件统计
  166. class FileStat():
  167.     global cos_log
  168.     def __init__(self):
  169.         self.delfilesuccnum = 0
  170.         self.deldirsuccnum = 0
  171.         self.delfilefailnum = 0
  172.         self.deldirfailnum = 0
  173.         self.lock = threading.Lock()
  174.     def addDelFileFailNum(self,num=1):
  175.         self.lock.acquire(1)
  176.         self.delfilefailnum += num
  177.         self.lock.release()
  178.     def addDelDirFailNum(self,num=1):
  179.         self.lock.acquire(1)
  180.         self.deldirfailnum += num
  181.         self.lock.release()
  182.     def addDelDirSuccNum(self, num=1):
  183.         self.lock.acquire(1)
  184.         self.deldirsuccnum += num
  185.         self.lock.release()
  186.     def addDelFileSuccNum(self, num=1):
  187.         self.lock.acquire(1)
  188.         self.delfilesuccnum += num
  189.         self.lock.release()
  190.     def printStat(self):
  191.         msg ="".join(["delfilesuccnum=",str(self.delfilesuccnum),
  192.                 ",delfilefailnum=",str(self.delfilefailnum),
  193.                 ",deldirsuccnum=",str(self.deldirsuccnum),
  194.                 ",deldirfailnum=",str(self.deldirfailnum)])
  195.         print(msg)
  196.     def logStat(self):
  197.         curtime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
  198.         log = ''.join(["delfilenum=",str(self.delfilesuccnum),
  199.             ",deldirnum=",str(self.deldirsuccnum),",delfilefailnum=",
  200.             str(self.delfilefailnum),",deldirfailnum=",str(self.deldirfailnum)])
  201.         cos_log.info(log)
  202. #执行时间统计
  203. class TimeStat(object):
  204.     global cos_log
  205.     def __init__(self):
  206.         self.start()
  207.     def start(self):
  208.         self.start = datetime.datetime.now()
  209.         self.t1 = time.time()
  210.         msg = "delete task started  ..........."
  211.         cos_log.info(msg)
  212.     def end(self):
  213.         self.end = datetime.datetime.now()
  214.         self.t2 = time.time()
  215.         msg = "delete task ended\n\nrm task finished,\ntimecost:"+str(self.t2-self.t1) + " (s)"
  216.         cos_log.info(msg)
  217. #删除文件列表中的文件
  218. def delfiles(cos_client, bucket, filelist):
  219.     for f in filelist:
  220.         delfile(cos_client, bucket, f)
  221. def delfolders(cos_client, bucket, folderlist):
  222.     for f in folderlist:
  223.         delfolder(cos_client, bucket, f)
  224. #文件夹删除
  225. def delfolder(cos_client, bucket, folder):
  226.     global stat
  227.     global cos_log
  228.     if not folder:
  229.         return 0
  230.     delfolderreq = DelFolderRequest(bucket, folder)
  231.     retry = 0
  232.     while (retry < MAX_RETRY_TIMES):
  233.         ret = cos_client.del_folder(delfolderreq)
  234.         msg = "delfolder fail, bucket="+bucket+",folder="+folder+ret['message']
  235.         if (ret['code'] == 0):
  236.             break
  237.         elif (ret['code'] == -166):
  238.             cos_log.warning(msg)
  239.             break
  240.         #操作太频繁,频控
  241.         elif (ret['code'] == -71):
  242.             sleep(random.randint(1,5))
  243.             cos_log.warning(msg)
  244.             retry += 1
  245.             continue
  246.         #文件夹非空
  247.         elif (ret['code'] == -173):
  248.             break
  249.         else:
  250.             cos_log.warning(msg)
  251.             retry += 1
  252.     if (ret['code'] != 0 and  ret['code'] != -166):
  253.         stat.addDelDirFailNum()
  254.         cos_log.error("delfolder fail, bucket="+bucket+",folder="+folder+ret['message'])
  255.         return ret['code']
  256.     if (ret['code'] == 0):
  257.         stat.addDelDirSuccNum()
  258.         msg = "delfolder success, bucket="+bucket+",folder="+folder
  259.         cos_log.info(msg)
  260.     return 0
  261. #文件删除
  262. def delfile(cos_client, bucket, filepath):
  263.     global stat
  264.     global cos_log
  265.     delfilereq = DelFileRequest(bucket, filepath)
  266.     retry = 0
  267.     while (retry < MAX_RETRY_TIMES):
  268.         ret = cos_client.del_file(delfilereq)
  269.         msg = "delfile fail bucket="+bucket+",file="+filepath+ret['message']
  270.         if (ret['code'] == 0):
  271.             break
  272.         #文件不存在
  273.         elif (ret['code'] == -166):
  274.             cos_log.warning(msg)
  275.             break
  276.         #单目录写操作过快
  277.         elif (ret['code'] == -143):
  278.             sleep(random.randint(1,5))
  279.             cos_log.warning(msg)
  280.             retry += 1
  281.             continue
  282.         #操作太频繁,频控
  283.         elif (ret['code'] == -71):
  284.             sleep(random.randint(1,5))
  285.             cos_log.warning(msg)
  286.             retry += 1
  287.             continue
  288.         else:
  289.             cos_log.warning(msg)
  290.             retry += 1
  291.             continue
  292.     if (ret['code'] != 0 and  ret['code'] != -166):
  293.         stat.addDelFileFailNum()
  294.         cos_log.error("delfile fail, bucket="+bucket+",file="+filepath+ret['message'])
  295.         return ret['code']
  296.     if (ret['code'] == 0):
  297.         stat.addDelFileSuccNum()
  298.         msg = "delfile success, bucket="+bucket+",file="+filepath
  299.         cos_log.info(msg)
  300.     return 0
  301. #递归文件夹进行文件删除
  302. def delete_r(cos_client, bucket, path, thread_pool_file):
  303.     global stat
  304.     global config
  305.     global cos_log
  306.     cos_log.debug("delete_r bucket:"+bucket+",path:"+path)
  307.     context = u""
  308.     #递归文件夹
  309.     while True:
  310.         listfolderreq = ListFolderRequest(bucket, path, 1000, u'', context)
  311.         retry = 0
  312.         while (retry < MAX_RETRY_TIMES):
  313.             listret = cos_client.list_folder(listfolderreq)
  314.             if listret['code'] != 0 :
  315.                 retry += 1
  316.                 sleep(random.randint(1,3))
  317.                 continue
  318.             else:
  319.                 break
  320.         if (listret['code'] != 0):
  321.             cos_log.error("delete_r: list folder fail:"+path +",return msg:"+ listret['message'])
  322.             return listret['code']
  323.         if (len(listret['data']['infos']) == 0):
  324.             break;
  325.         filelist = []
  326.         dirlist = []
  327.         for info in listret['data']['infos']:
  328.             fullname = path + info['name']
  329.             #list出来的文件列表中文件夹和文件本身是混杂一起的
  330.             if info.has_key('filesize'):
  331.                 filelist.append(fullname)
  332.                 if (len(filelist) >= ONE_TASK_DEL_FILE_NUMS):
  333.                     args = [cos_client, bucket, filelist]
  334.                     args_tuple = (args,None)
  335.                     args_list = [args_tuple]
  336.                     requests = threadpool.makeRequests(delfiles, args_list)
  337.                     for req in requests:
  338.                         thread_pool_file.putRequest(req)
  339.                         filelist = []
  340.                         continue
  341.                 else:
  342.                     pass
  343.             else:
  344.                 dirlist.append(fullname)
  345.                 if (len(dirlist) >= ONE_TASK_DEL_FILE_NUMS):
  346.                     args = [cos_client, bucket, dirlist]
  347.                     args_tuple = (args,None)
  348.                     args_list = [args_tuple]
  349.                     requests = threadpool.makeRequests(delfolders, args_list)
  350.                     for req in requests:
  351.                         thread_pool_file.putRequest(req)
  352.                         dirlist = []
  353.                         continue
  354.                 else:
  355.                     pass
  356.                 pass
  357.         if (len(filelist) > 0):
  358.             args = [cos_client, bucket, filelist]
  359.             args_tuple = (args,None)
  360.             args_list = [args_tuple]
  361.             requests = threadpool.makeRequests(delfiles, args_list)
  362.             for req in requests:
  363.                 thread_pool_file.putRequest(req)
  364.                 filelist = []
  365.         else:
  366.             pass
  367.         if (len(dirlist) > 0):
  368.             args = [cos_client, bucket, dirlist]
  369.             args_tuple = (args,None)
  370.             args_list = [args_tuple]
  371.             requests = threadpool.makeRequests(delfolders, args_list)
  372.             for req in requests:
  373.                 thread_pool_file.putRequest(req)
  374.                 filelist = []
  375.         else:
  376.             pass
  377.         cos_log.debug("delete_r thread pool file waiting\n")
  378.         thread_pool_file.wait()
  379.         cos_log.debug("delete_r thread pool file waiting end\n")
  380.         if (listret['data']['listover'] == False):
  381.             context = listret['data']['context']
  382.             continue
  383.         else:
  384.             break
  385.     stat.logStat()
  386.     return 0
  387. #支持Ctrl+C终止程序
  388. class Watcher():
  389.     def __init__(self):
  390.         self.child = os.fork()
  391.         if self.child == 0:
  392.             return
  393.         else:
  394.             self.watch()
  395.     def watch(self):
  396.         global cos_log
  397.         try:
  398.             os.wait()
  399.         except KeyboardInterrupt:
  400.             cos_log.ERROR("ctrl+c terminated rm_recursive.py, exiting...")
  401.             self.kill()
  402.         sys.exit()
  403.     def kill(self):
  404.         try:
  405.             os.kill(self.child, signal.SIGKILL)
  406.         except OSError:
  407.             pass
  408. def cmd_rm(COSDIR):
  409.     global thread_pool
  410.     global cos_log
  411.     global stat
  412.     cos_log = loginit()
  413.     stat = FileStat()
  414.     timestat = TimeStat()
  415.     if HAS_FORK:
  416.       Watcher()
  417.     path = COSDIR.decode('utf-8')
  418.     thread_pool_dir = threadpool.ThreadPool(dir_thread_num)
  419.     thread_pool_file = threadpool.ThreadPool(file_thread_num)
  420.     cos_log.debug("bucket:"+bucket +",path:"+path)
  421.     args = [cos_client, bucket, path, thread_pool_file]
  422.     args_tuple = (args, None)
  423.     args_list = [args_tuple]
  424.     requests = threadpool.makeRequests(delete_r, args_list)
  425.     for req in requests:
  426.         thread_pool_dir.putRequest(req)
  427.     cos_log.debug("thread_pool_dir waiting.....\n")
  428.     thread_pool_dir.wait()
  429.     thread_pool_dir.dismissWorkers(dir_thread_num, True)
  430.     cos_log.debug("thread_pool_dir wait end.....\n")
  431.     timestat.end()
  432.     stat.logStat()
  433. if sys.argv[1] in ['config','ls','mkdir','put','rm','delete','del'] and len(sys.argv) >= 3:
  434.     if sys.argv[1] == 'config':
  435.         parser = OptionParser()
  436.         parser.add_option("-a", "--appid", dest="appid", help="specify appid")
  437.         parser.add_option("-i", "--id", dest="secret_id", help="specify secret id")
  438.         parser.add_option("-k", "--key", dest="secret_key", help="specify secret key")
  439.         parser.add_option("-r", "--region", dest="region", help="specify region")
  440.         parser.add_option("-b", "--bucket", dest="bucket", help="specify bucket")
  441.         (options, args) = parser.parse_args()
  442.         CMD_LIST['config'] = cmd_configure
  443.         CMD_LIST['config'](args, options)
  444.     if sys.argv[1] == 'ls':
  445.         cmd_loadconfigure()
  446.         cos_client = CosClient(appid, secret_id, secret_key, region)
  447.         COSDIR = sys.argv[2]
  448.         cmd_lsdir(COSDIR)
  449.     if sys.argv[1] == 'mkdir':
  450.         cmd_loadconfigure()
  451.         cos_client = CosClient(appid, secret_id, secret_key, region)
  452.         COSDIR = sys.argv[2]
  453.         cmd_mkdir(COSDIR)
  454.     if sys.argv[1] == 'put' and len(sys.argv) == 4:
  455.         cmd_loadconfigure()
  456.         cos_client = CosClient(appid, secret_id, secret_key, region)
  457.         LOCALFILE = sys.argv[2]
  458.         COSFILE = sys.argv[3]
  459.         cmd_put(LOCALFILE,COSFILE)
  460.     if sys.argv[1] in ('rm','delete','del'):
  461.         cmd_loadconfigure()
  462.         cos_client = CosClient(appid, secret_id, secret_key, region)
  463.         COSDIR = sys.argv[2]
  464.         path = COSDIR.decode('utf-8')
  465.         cmd_rm(path)
  466. else:
  467.     print(HELP)
  468.     exit()

Fri Mar 31 21:51:20 CST 2017

avatar

发表评论

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

目前评论:13   其中:访客  9   博主  4

    • avatar czwzgs 0

      想请问能否自动把备份的/data/backup/下面的所有.tgz文件批量上传,完成后自动删除.tgz备份文件?
      最后说一句,oneinstack实在太牛逼了。。感谢万能的yeho

      • avatar k 0

        Python 2.7运行提示

        Traceback (most recent call last):
        File “./coscmd”, line 18, in
        from cos import CosClient
        File “/home/lnmp-master/tools/cos/__init__.py”, line 4, in
        from qcloud_cos import CosClient
        File “/home/lnmp-master/tools/cos/qcloud_cos/__init__.py”, line 3, in
        from .cos_client import CosClient
        File “/home/lnmp-master/tools/cos/qcloud_cos/cos_client.py”, line 4, in
        import requests
        ImportError: No module named requests

          • avatar yeho Admin

            @k 依赖requests ,pip install requests试试

          • avatar 请输入您的QQ号 2

            貌似不能删除文件夹里面还有文件夹的。这就尴尬了。删除带文件夹里面还有文件夹的会显示:ERROR delfolder fail, bucket=这里是我的backup名称,folder=/js-demo/automatic-switching/js/url:http://sh.file.myqcloud.com/files/v2/1251630154/mrjucn/js-demo/automatic-switching/js/, status_code:404

            • avatar Baby Q 0

              你好厉害的。要是要一个一个删。几万张图片。一辈子都删不完。 :oops:

              • avatar 请输入您的QQ号 3

                刚发现,我的友链已经被清理了 :cry:

                • avatar 请输入您的QQ号 3

                  博主好人,谢谢提供!另外,我博客换了域名了,有时间请更新下友链吧 :grin:

                  • avatar FGG 1

                    oneinstack可以安装Python2.7吗……
                    我怎么不知道……
                    求方法

                      • avatar yeho Admin

                        @FGG ./addons.sh 选择7

                          • avatar FGG 1

                            @yeho 所以说Python2.7是给certbot提供环境顺便安装的(⁄ ⁄•⁄ω⁄•⁄ ⁄)