I have been able to successfully upload some of the tarballs using python3 requests.post, but some of the tarballs are too big and I get a MemoryError from python. Through some research, I found the best solution for my situation would be to chunk the tarballs into smaller, multipart uploads. It looks like the script is unable to upload the chunks successfully. I am getting the following error: <Response [400]>
: <Response [400]>, Content-Range: bytes 7294976-7299071/89623040
So it is obviously the header. Is there an appropriate header or a better way to upload large files with python?
import glob
import requests
import sys
import os
PWD = sys.argv[1]
NEXUS_URL = sys.argv[2]
NEXUS_USER = sys.argv[3]
NEXUS_PASS = sys.argv[4]
NEXUS_REPO = sys.argv[5]
DEFAULT_VERSION = sys.argv[6]
CHUNK_SIZE = 4096
def get_image_list(pwd):
tar_list = glob.glob('{}/images/*.tar'.format(pwd))
return tar_list
def read_in_chunks(file_object, chunk_size):
while True:
data = file_object.read(chunk_size)
if not data:
break
yield data
def upload_image(pwd, image, url, nexus_user, nexus_pass, nexus_repo, default_version, chunk_size, nexus_auth_string):
index = 0
offset = 0
headers = {}
content_name = str(image).split('/', 1)[1]
content_path = os.path.abspath(image)
content_size = os.stat(content_path).st_size
print(content_name, content_path, content_size)
file_object = open(content_path, 'rb')
for chunk in read_in_chunks(file_object, chunk_size):
offset = index + len(chunk)
headers['Content-Range'] = 'bytes %s-%s/%s' % (index,
offset - 1, content_size)
index = offset
nexus_auth = requests.auth.HTTPBasicAuth(nexus_user, nexus_pass)
params = (
('repository', nexus_repo),
)
file = {'raw.directory': (None, default_version),
'raw.asset1': (None, chunk),
'raw.asset1.filename': (None, image)}
r = requests.post(url, auth=nexus_auth, files=file, params=params,
headers=headers, verify=False)
print(r)
print("r: %s, Content-Range: %s" % (r, headers['Content-Range']))
def push_to_nexus(pwd, nexus_url, nexus_user, nexus_pass, nexus_repo, default_version, chunk_size):
images_list = get_image_list(pwd)
image_list = [i.split('images/', 1)[1] for i in images_list]
for image in image_list:
upload_image(pwd, image, nexus_url, nexus_user, nexus_pass, nexus_repo, default_version, chunk_size)
if __name__ == '__main__':
push_to_nexus(PWD, NEXUS_URL, NEXUS_USER, NEXUS_PASS, NEXUS_REPO, DEFAULT_VERSION, CHUNK_SIZE)