- A+
所属分类：python
要将upyun上存储的文件迁移到七牛存储上，不管出于何种目的，多多少少会遇到这种需求的。
方法如下：
- 列出upyun空间上所有文件
- 通过七牛提供的qfetch工具上传到七牛
列出upyun文件
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 |
vi upyun_list.py #! /usr/bin/env python # -*- coding: utf-8 -*- from base64 import b64encode import requests import urllib import Queue import sys bucket = sys.argv[1] username = 'ttlsa' password = 'www.ttlsa.com' queue = Queue.LifoQueue() queue_list = Queue.LifoQueue() count = 0 def do_http_request(method, key, upyun_iter): uri = '/' + bucket + (lambda x: x[0] == '/' and x or '/' + x)(key) if isinstance(uri, unicode): uri = uri.encode('utf-8') uri = urllib.quote(uri) headers = {} headers['Authorization'] = "Basic " + b64encode(username + ':' + password) headers['User-Agent'] = "uptechs" headers['X-List-Limit'] = 300 if method is not 'DELETE': if upyun_iter is not None or upyun_iter is not 'g2gCZAAEbmV4dGQAA2VvZg': headers['x-list-iter'] = upyun_iter url = "http://v0.api.upyun.com" + uri requests.adapters.DEFAULT_RETRIES = 5 session = requests.session() try: response = session.request(method, url, headers=headers, timeout=30) status = response.status_code if status == 200 and method != 'DELETE': content = response.content try: iter_header = response.headers['x-upyun-list-iter'] except Exception as e: iter_header = 'g2gCZAAEbmV4dGQAA2VvZg' return content + "`" + str(iter_header) elif status == 200 and method == 'DELETE': return True else: print 'status: ' + str(status) + '--->' + url print 'message: ' + str(response.headers['X-Error-Code']) print 'message: ' + str(response.content) return None except Exception as e: pass def getlist(key, upyun_iter): content = do_http_request('GET', key, upyun_iter) if not content: return None content = content.split("`") items = content[0].split('\n') content = [dict(zip(['name', 'type', 'size', 'time'], x.split('\t'))) for x in items] + content[1].split() return content def print_file_with_iter(path): upyun_iter = None while True: while upyun_iter != 'g2gCZAAEbmV4dGQAA2VvZg': res = getlist(path, upyun_iter) if res: upyun_iter = res[-1] for i in res[:-1]: try: if not i['name']: continue new_path = path + i['name'] if path == '/' else path + '/' + i['name'] if i['type'] == 'F': queue.put(new_path) queue_list.put(new_path) elif i['type'] == 'N': print sys.argv[3] + new_path with open(bucket + '_file.txt', 'a') as f: f.write(sys.argv[3] + new_path + '\n') except Exception as e: print e else: if not queue.empty(): path = queue.get() upyun_iter = None queue.task_done() else: break if __name__ == '__main__': path = sys.argv[2] print_file_with_iter(path) |
执行upyun_list.py
# python upyun_list.py ttlsa-files / http://imgs.ttlsa.com
输出的内容格式如下
http://imgs.ttlsa.com/201601/142135564177453.png
http://imgs.ttlsa.com/201601/142137469838888.png
http://imgs.ttlsa.com/201601/142137554414262.png
同时，在本地目录保存在ttlsa-files_file.txt文件中。
上传到七牛
|
1 |
# qfetch_linux_amd64 -ak=$qiniu_ak -sk=$qiniu_sk -bucket="upyun-ttlsa-files" -file="ttlsa-files_file.txt" -job="upyun_ttlsa-files" -worker=2 |
qfetch具体选项要参见七牛提供的文档。http://developer.qiniu.com/code/v6/tool/qfetch.html
微信公众号
扫一扫关注运维生存时间公众号，获取最新技术文章~
