Skip to content

P
python-poetry
Commit c63da256 by Sébastien Eustace
Options

Refactor the publishing module

parent f4e02009
Showing with 309 additions and 270 deletions
poetry/masonry/publishing/publisher.py
import hashlib
import io
import re
import requests
import toml
from pathlib import Path
from requests import adapters
from requests.exceptions import HTTPError
from requests.packages.urllib3 import util
from requests_toolbelt import user_agent
from requests_toolbelt.multipart import (
MultipartEncoder, MultipartEncoderMonitor
)
from poetry import __version__
from poetry.locations import CONFIG_DIR
from ..metadata import Metadata
wheel_file_re = re.compile(
r"""^(?P<namever>(?P<name>.+?)(-(?P<ver>\d.+?))?)
((-(?P<build>\d.*?))?-(?P<pyver>.+?)-(?P<abi>.+?)-(?P<plat>.+?)
\.whl|\.dist-info)$""",
re.VERBOSE
)
KEYWORDS_TO_NOT_FLATTEN = {'gpg_signature', 'content'}
from .uploader import Uploader
class Publisher:
......@@ -40,6 +16,7 @@ class Publisher:
self._poetry = poetry
self._package = poetry.package
self._io = io
self._uploader = Uploader(poetry, io)
def publish(self, repository_name):
if repository_name:
......@@ -106,251 +83,8 @@ class Publisher:
if not password:
password = self._io.ask_hidden('Password:')
session = requests.session()
session.auth = (username, password)
session.headers['User-Agent'] = self._make_user_agent_string()
for scheme in ('http://', 'https://'):
session.mount(scheme, self._make_adapter_with_retries())
# TODO: handle certificates
try:
self.upload(session, url)
finally:
session.close()
def register(self, session, url):
"""
Register a package to a repository.
"""
dist = self._poetry.file.parent / 'dist'
file = dist / '{}-{}.tar.gz'.format(
self._package.name, self._package.version
)
if not file.exists():
raise RuntimeError(
'"{0}" does not exist on the file system.'.format(file.name)
)
data = self.post_data(file)
data.update({
":action": "submit",
"protocol_version": "1",
})
data_to_send = self._convert_data_to_list_of_tuples(data)
encoder = MultipartEncoder(data_to_send)
resp = session.post(
url,
data=encoder,
allow_redirects=False,
headers={'Content-Type': encoder.content_type},
)
return resp
def upload(self, session, url):
"""
Upload packages for the current project.
"""
try:
self._upload(session, url)
except HTTPError as e:
if (
e.response.status_code not in (403, 400)
or e.response.status_code == 400 and 'was ever registered' not in e.response.text
):
raise
# It may be the first time we publish the package
# We'll try to register it and go from there
try:
self.register(session, url)
except HTTPError:
raise
def post_data(self, file):
meta = Metadata.from_package(self._package)
file_type = self._get_type(file)
blake2_256_hash = hashlib.blake2b(digest_size=256 // 8)
md5_hash = hashlib.md5()
sha2_hash = hashlib.sha256()
with file.open('rb') as fp:
for content in iter(lambda: fp.read(io.DEFAULT_BUFFER_SIZE), b''):
md5_hash.update(content)
sha2_hash.update(content)
blake2_256_hash.update(content)
md5_digest = md5_hash.hexdigest()
sha2_digest = sha2_hash.hexdigest()
blake2_256_digest = blake2_256_hash.hexdigest()
if file_type == 'bdist_wheel':
wheel_info = wheel_file_re.match(file.name)
py_version = wheel_info.group("pyver")
else:
py_version = None
data = {
# identify release
"name": meta.name,
"version": meta.version,
# file content
"filetype": file_type,
"pyversion": py_version,
# additional meta-data
"metadata_version": meta.metadata_version,
"summary": meta.summary,
"home_page": meta.home_page,
"author": meta.author,
"author_email": meta.author_email,
"maintainer": meta.maintainer,
"maintainer_email": meta.maintainer_email,
"license": meta.license,
"description": meta.description,
"keywords": meta.keywords,
"platform": meta.platforms,
"classifiers": meta.classifiers,
"download_url": meta.download_url,
"supported_platform": meta.supported_platforms,
"comment": None,
"md5_digest": md5_digest,
"sha256_digest": sha2_digest,
"blake2_256_digest": blake2_256_digest,
# PEP 314
"provides": meta.provides,
"requires": meta.requires,
"obsoletes": meta.obsoletes,
# Metadata 1.2
"project_urls": meta.project_urls,
"provides_dist": meta.provides_dist,
"obsoletes_dist": meta.obsoletes_dist,
"requires_dist": meta.requires_dist,
"requires_external": meta.requires_external,
"requires_python": meta.requires_python,
}
# Metadata 2.1
if meta.description_content_type:
data['description_content_type'] = meta.description_content_type
# TODO: Provides extra
return data
def _upload(self, session, url):
dist = self._poetry.file.parent / 'dist'
packages = dist.glob(
'{}-{}*'.format(self._package.name, self._package.version)
)
files = (
i for i in packages if (
i.match(
'{}-{}-*.whl'.format(
self._package.name, self._package.version
)
)
or
i.match(
'{}-{}.tar.gz'.format(
self._package.name, self._package.version
)
)
)
)
for file in files:
# TODO: Check existence
resp = self._upload_file(session, url, file)
resp.raise_for_status()
def _upload_file(self, session, url, file):
data = self.post_data(file)
data.update({
# action
":action": "file_upload",
"protocol_version": "1",
})
data_to_send = self._convert_data_to_list_of_tuples(data)
with file.open('rb') as fp:
data_to_send.append((
"content",
(file.name, fp, "application/octet-stream"),
))
encoder = MultipartEncoder(data_to_send)
bar = self._io.create_progress_bar(encoder.len)
bar.set_format(
" - Uploading <info>{0}</> <comment>%percent%%</>".format(
file.name
)
)
monitor = MultipartEncoderMonitor(
encoder, lambda monitor: bar.set_progress(monitor.bytes_read)
)
bar.start()
resp = session.post(
url,
data=monitor,
allow_redirects=False,
headers={'Content-Type': monitor.content_type}
)
if resp.ok:
bar.finish()
self._io.writeln('')
else:
self._io.overwrite('')
return resp
def _convert_data_to_list_of_tuples(self, data):
data_to_send = []
for key, value in data.items():
if (key in KEYWORDS_TO_NOT_FLATTEN or
not isinstance(value, (list, tuple))):
data_to_send.append((key, value))
else:
for item in value:
data_to_send.append((key, item))
return data_to_send
def _get_type(self, file):
exts = file.suffixes
if exts[-1] == '.whl':
return 'bdist_wheel'
elif len(exts) >= 2 and ''.join(exts[-2:]) == '.tar.gz':
return 'sdist'
raise ValueError(
'Unknown distribution format {}'.format(''.join(exts))
)
@staticmethod
def _make_adapter_with_retries():
retry = util.Retry(
connect=5,
total=10,
method_whitelist=['GET'],
status_forcelist=[500, 501, 502, 503],
)
return adapters.HTTPAdapter(max_retries=retry)
self._uploader.auth(username, password)
@staticmethod
def _make_user_agent_string():
return user_agent(
'poetry', __version__,
)
return self._uploader.upload(url)
poetry/masonry/publishing/uploader.py 0 → 100644
import hashlib
import io
import re
import requests
from requests import adapters
from requests.exceptions import HTTPError
from requests.packages.urllib3 import util
from requests_toolbelt import user_agent
from requests_toolbelt.multipart import (
MultipartEncoder, MultipartEncoderMonitor
)
from poetry.__version__ import __version__
from ..metadata import Metadata
wheel_file_re = re.compile(
r"""^(?P<namever>(?P<name>.+?)(-(?P<ver>\d.+?))?)
((-(?P<build>\d.*?))?-(?P<pyver>.+?)-(?P<abi>.+?)-(?P<plat>.+?)
\.whl|\.dist-info)$""",
re.VERBOSE
)
KEYWORDS_TO_NOT_FLATTEN = {'gpg_signature', 'content'}
_has_blake2 = hasattr(hashlib, 'blake2b')
class Uploader:
def __init__(self, poetry, io):
self._poetry = poetry
self._package = poetry.package
self._io = io
self._username = None
self._password = None
@property
def user_agent(self):
return user_agent('poetry', __version__)
@property
def adapter(self):
retry = util.Retry(
connect=5,
total=10,
method_whitelist=['GET'],
status_forcelist=[500, 501, 502, 503],
)
return adapters.HTTPAdapter(max_retries=retry)
def auth(self, username, password):
self._username = username
self._password = password
def make_session(self):
session = requests.session()
if self.is_authenticated():
session.auth = (self._username, self._password)
session.headers['User-Agent'] = self.user_agent
for scheme in ('http://', 'https://'):
session.mount(scheme, self.adapter)
return session
def is_authenticated(self):
return self._username is not None and self._password is not None
def upload(self, url):
session = self.make_session()
try:
self._upload(session, url)
finally:
session.close()
def post_data(self, file):
meta = Metadata.from_package(self._package)
file_type = self._get_type(file)
if _has_blake2:
blake2_256_hash = hashlib.blake2b(digest_size=256 // 8)
md5_hash = hashlib.md5()
sha256_hash = hashlib.sha256()
with file.open('rb') as fp:
for content in iter(lambda: fp.read(io.DEFAULT_BUFFER_SIZE), b''):
md5_hash.update(content)
sha256_hash.update(content)
if _has_blake2:
blake2_256_hash.update(content)
md5_digest = md5_hash.hexdigest()
sha2_digest = sha256_hash.hexdigest()
if _has_blake2:
blake2_256_digest = blake2_256_hash.hexdigest()
else:
blake2_256_digest = None
if file_type == 'bdist_wheel':
wheel_info = wheel_file_re.match(file.name)
py_version = wheel_info.group("pyver")
else:
py_version = None
data = {
# identify release
"name": meta.name,
"version": meta.version,
# file content
"filetype": file_type,
"pyversion": py_version,
# additional meta-data
"metadata_version": meta.metadata_version,
"summary": meta.summary,
"home_page": meta.home_page,
"author": meta.author,
"author_email": meta.author_email,
"maintainer": meta.maintainer,
"maintainer_email": meta.maintainer_email,
"license": meta.license,
"description": meta.description,
"keywords": meta.keywords,
"platform": meta.platforms,
"classifiers": meta.classifiers,
"download_url": meta.download_url,
"supported_platform": meta.supported_platforms,
"comment": None,
"md5_digest": md5_digest,
"sha256_digest": sha2_digest,
"blake2_256_digest": blake2_256_digest,
# PEP 314
"provides": meta.provides,
"requires": meta.requires,
"obsoletes": meta.obsoletes,
# Metadata 1.2
"project_urls": meta.project_urls,
"provides_dist": meta.provides_dist,
"obsoletes_dist": meta.obsoletes_dist,
"requires_dist": meta.requires_dist,
"requires_external": meta.requires_external,
"requires_python": meta.requires_python,
}
# Metadata 2.1
if meta.description_content_type:
data['description_content_type'] = meta.description_content_type
# TODO: Provides extra
return data
def _upload(self, session, url):
try:
self._do_upload(session, url)
except HTTPError as e:
if (
e.response.status_code not in (403, 400)
or e.response.status_code == 400 and 'was ever registered' not in e.response.text
):
raise
# It may be the first time we publish the package
# We'll try to register it and go from there
try:
self._register(session, url)
except HTTPError:
raise
def _do_upload(self, session, url):
dist = self._poetry.file.parent / 'dist'
packages = dist.glob(
'{}-{}*'.format(self._package.name, self._package.version)
)
files = (
i for i in packages if (
i.match(
'{}-{}-*.whl'.format(
self._package.name, self._package.version
)
)
or
i.match(
'{}-{}.tar.gz'.format(
self._package.name, self._package.version
)
)
)
)
for file in files:
# TODO: Check existence
resp = self._upload_file(session, url, file)
resp.raise_for_status()
def _upload_file(self, session, url, file):
data = self.post_data(file)
data.update({
# action
":action": "file_upload",
"protocol_version": "1",
})
data_to_send = self._prepare_data(data)
with file.open('rb') as fp:
data_to_send.append((
"content",
(file.name, fp, "application/octet-stream"),
))
encoder = MultipartEncoder(data_to_send)
bar = self._io.create_progress_bar(encoder.len)
bar.set_format(
" - Uploading <info>{0}</> <comment>%percent%%</>".format(
file.name
)
)
monitor = MultipartEncoderMonitor(
encoder, lambda monitor: bar.set_progress(monitor.bytes_read)
)
bar.start()
resp = session.post(
url,
data=monitor,
allow_redirects=False,
headers={'Content-Type': monitor.content_type}
)
if resp.ok:
bar.finish()
self._io.writeln('')
else:
self._io.overwrite('')
return resp
def _register(self, session, url):
"""
Register a package to a repository.
"""
dist = self._poetry.file.parent / 'dist'
file = dist / '{}-{}.tar.gz'.format(
self._package.name, self._package.version
)
if not file.exists():
raise RuntimeError(
'"{0}" does not exist.'.format(file.name)
)
data = self.post_data(file)
data.update({
":action": "submit",
"protocol_version": "1",
})
data_to_send = self._prepare_data(data)
encoder = MultipartEncoder(data_to_send)
resp = session.post(
url,
data=encoder,
allow_redirects=False,
headers={'Content-Type': encoder.content_type},
)
return resp
def _prepare_data(self, data):
data_to_send = []
for key, value in data.items():
if (key in KEYWORDS_TO_NOT_FLATTEN or
not isinstance(value, (list, tuple))):
data_to_send.append((key, value))
else:
for item in value:
data_to_send.append((key, item))
return data_to_send
def _get_type(self, file):
exts = file.suffixes
if exts[-1] == '.whl':
return 'bdist_wheel'
elif len(exts) >= 2 and ''.join(exts[-2:]) == '.tar.gz':
return 'sdist'
raise ValueError(
'Unknown distribution format {}'.format(''.join(exts))
)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment