Skip to content

P
python-poetry
Commit ce69554f by Sébastien Eustace
Options

No longer depend on twine to upload

parent 692fe626
Showing with 439 additions and 266 deletions
poetry/console/commands/publish.py
from poetry.masonry.publisher import Publisher from poetry.masonry.publishing.publisher import Publisher
from .command import Command from .command import Command
......
poetry/masonry/metadata.py 0 → 100644
from poetry.semver.constraints import MultiConstraint
from poetry.utils.helpers import canonicalize_name
class Metadata:
metadata_version = '1.2'
# version 1.0
name = None
version = None
platforms = ()
supported_platforms = ()
summary = None
description = None
keywords = None
home_page = None
download_url = None
author = None
author_email = None
license = None
# version 1.1
classifiers = ()
requires = ()
provides = ()
obsoletes = ()
# version 1.2
maintainer = None
maintainer_email = None
requires_python = None
requires_external = ()
requires_dist = ()
provides_dist = ()
obsoletes_dist = ()
project_urls = ()
@classmethod
def from_package(cls, package) -> 'Metadata':
meta = cls()
meta.name = canonicalize_name(package.name)
meta.version = package.version
meta.summary = package.description
meta.description = package.readme
meta.keywords = ','.join(package.keywords)
meta.home_page = package.homepage or package.repository_url
meta.author = package.author_name
meta.author_email = package.author_email
meta.license = package.license
meta.classifiers = package.classifiers
# Version 1.1
meta.requires = [d.to_pep_508() for d in package.requires]
# Version 1.2
meta.maintainer = meta.author
meta.maintainer_email = meta.author_email
meta.requires_python = package.python_constraint
meta.requires_dist = [d.to_pep_508() for d in package.requires]
# Requires python
constraint = package.python_constraint
if isinstance(constraint, MultiConstraint):
python_requires = ','.join(
[str(c).replace(' ', '') for c in constraint.constraints]
)
else:
python_requires = str(constraint).replace(' ', '')
meta.requires_python = python_requires
return meta
poetry/masonry/publisher.py deleted 100644 → 0
# -*- coding: utf-8 -*-
import toml
import twine.utils
from pathlib import Path
from requests.exceptions import HTTPError
from twine.commands.upload import find_dists, skip_upload
from twine.repository import Repository as BaseRepository
from twine.exceptions import PackageNotFound, RedirectDetected
from twine.package import PackageFile
from requests_toolbelt.multipart import (
MultipartEncoder, MultipartEncoderMonitor
)
from poetry.locations import CONFIG_DIR
class Repository(BaseRepository):
def __init__(self, io, url, username, password):
self._io = io
super(Repository, self).__init__(url, username, password)
def register(self, package):
data = package.metadata_dictionary()
data.update({
":action": "submit",
"protocol_version": "1",
})
self._io.writeln(
" - Registering <info>{0}</>".format(package.basefilename)
)
data_to_send = self._convert_data_to_list_of_tuples(data)
encoder = MultipartEncoder(data_to_send)
resp = self.session.post(
self.url,
data=encoder,
allow_redirects=False,
headers={'Content-Type': encoder.content_type}
)
# Bug 28. Try to silence a ResourceWarning by releasing the socket.
resp.close()
return resp
def _upload(self, package):
data = package.metadata_dictionary()
data.update({
# action
":action": "file_upload",
"protocol_version": "1",
})
data_to_send = self._convert_data_to_list_of_tuples(data)
with open(package.filename, "rb") as fp:
data_to_send.append((
"content",
(package.basefilename, fp, "application/octet-stream"),
))
encoder = MultipartEncoder(data_to_send)
bar = self._io.create_progress_bar(encoder.len)
bar.set_format(
" - Uploading <info>{0}</> <comment>%percent%%</>".format(
package.basefilename
)
)
monitor = MultipartEncoderMonitor(
encoder, lambda monitor: bar.set_progress(monitor.bytes_read)
)
bar.start()
resp = self.session.post(
self.url,
data=monitor,
allow_redirects=False,
headers={'Content-Type': monitor.content_type}
)
if resp.ok:
bar.finish()
self._io.writeln('')
else:
self._io.overwrite('')
return resp
class Publisher(object):
"""
Registers and publishes packages to remote repositories.
"""
def __init__(self, poetry, io):
self._poetry = poetry
self._package = poetry.package
self._io = io
def publish(self, repository_name):
if repository_name:
self._io.writeln(
f'Publishing <info>{self._package.pretty_name}</info> '
f'(<comment>{self._package.pretty_version}</comment>) '
f'to <fg=cyan>{repository_name}</>'
)
else:
self._io.writeln(
f'Publishing <info>{self._package.pretty_name}</info> '
f'(<comment>{self._package.pretty_version}</comment>) '
f'to <fg=cyan>PyPI</>'
)
if not repository_name:
url = 'https://upload.pypi.org/legacy/'
else:
# Retrieving config information
config_file = Path(CONFIG_DIR) / 'config.toml'
if not config_file.exists():
raise RuntimeError(
'Config file does not exist. '
'Unable to get repository information'
)
with config_file.open() as f:
config = toml.loads(f.read())
if (
'repositories' not in config
or repository_name not in config['repositories']
):
raise RuntimeError(
f'Repository {repository_name} is not defined'
)
url = config['repositories'][repository_name]['url']
username = None
password = None
auth_file = Path(CONFIG_DIR) / 'auth.toml'
if auth_file.exists():
with auth_file.open() as f:
auth_config = toml.loads(f.read())
if 'http-basic' in auth_config and repository_name in auth_config['http-basic']:
config = auth_config['http-basic'][repository_name]
username = config.get('username')
password = config.get('password')
# Requesting missing credentials
if not username:
username = self._io.ask('Username:')
if not password:
password = self._io.ask_hidden('Password:')
repository = Repository(self._io, url, username, password)
# TODO: handle certificates
self.upload(repository)
def register(self, repository):
"""
Register a package to a repository.
"""
dist = self._poetry.file.parent / 'dist'
package = dist / f'{self._package.name}-{self._package.version}.tar.gz'
if package.exists():
raise PackageNotFound(
'"{0}" does not exist on the file system.'.format(package)
)
resp = repository.register(
PackageFile.from_filename(str(package), None)
)
repository.close()
if resp.is_redirect:
raise RedirectDetected(
('"{0}" attempted to redirect to "{1}" during registration.'
' Aborting...').format(repository.url,
resp.headers["location"]))
resp.raise_for_status()
def upload(self, repository):
"""
Upload packages for the current project.
"""
try:
self._upload(repository)
except HTTPError as e:
if (
e.response.status_code not in (403, 400)
or e.response.status_code == 400 and 'was ever registered' not in e.response.text
):
raise
# It may be the first time we publish the package
# We'll try to register it and go from there
try:
self.register(repository)
except HTTPError:
raise
def _upload(self, repository):
skip_existing = False
dist = self._poetry.file.parent / 'dist'
packages = list(dist.glob(f'{self._package.name}-{self._package.version}*'))
dists = find_dists([str(p) for p in packages])
uploads = [i for i in dists if not i.endswith(".asc")]
for filename in uploads:
package = PackageFile.from_filename(filename, None)
skip_message = (
" - Skipping <comment>{0}</> because it appears to already exist"
.format(
package.basefilename
)
)
# Note: The skip_existing check *needs* to be first, because otherwise
# we're going to generate extra HTTP requests against a hardcoded
# URL for no reason.
if skip_existing and repository.package_is_uploaded(package):
self._io.writeln(skip_message)
continue
resp = repository.upload(package)
# Bug 92. If we get a redirect we should abort because something seems
# funky. The behaviour is not well defined and redirects being issued
# by PyPI should never happen in reality. This should catch malicious
# redirects as well.
if resp.is_redirect:
raise RedirectDetected(
('"{0}" attempted to redirect to "{1}" during upload.'
' Aborting...').format(repository.url,
resp.headers["location"]))
if skip_upload(resp, skip_existing, package):
self._io.writeln(skip_message)
continue
twine.utils.check_status_code(resp)
# Bug 28. Try to silence a ResourceWarning by clearing the connection
# pool.
repository.close()
poetry/masonry/publishing/__init__.py 0 → 100644
from .publisher import Publisher
poetry/masonry/publishing/publisher.py 0 → 100644
import hashlib
import io
import re
import requests
import toml
from pathlib import Path
from requests import adapters
from requests.exceptions import HTTPError
from requests.packages.urllib3 import util
from requests_toolbelt import user_agent
from requests_toolbelt.multipart import (
MultipartEncoder, MultipartEncoderMonitor
)
from poetry import __version__
from poetry.locations import CONFIG_DIR
from ..metadata import Metadata
wheel_file_re = re.compile(
r"""^(?P<namever>(?P<name>.+?)(-(?P<ver>\d.+?))?)
((-(?P<build>\d.*?))?-(?P<pyver>.+?)-(?P<abi>.+?)-(?P<plat>.+?)
\.whl|\.dist-info)$""",
re.VERBOSE
)
KEYWORDS_TO_NOT_FLATTEN = {'gpg_signature', 'content'}
class Publisher:
"""
Registers and publishes packages to remote repositories.
"""
def __init__(self, poetry, io):
self._poetry = poetry
self._package = poetry.package
self._io = io
def publish(self, repository_name):
if repository_name:
self._io.writeln(
f'Publishing <info>{self._package.pretty_name}</info> '
f'(<comment>{self._package.pretty_version}</comment>) '
f'to <fg=cyan>{repository_name}</>'
)
else:
self._io.writeln(
f'Publishing <info>{self._package.pretty_name}</info> '
f'(<comment>{self._package.pretty_version}</comment>) '
f'to <fg=cyan>PyPI</>'
)
if not repository_name:
url = 'https://upload.pypi.org/legacy/'
else:
# Retrieving config information
config_file = Path(CONFIG_DIR) / 'config.toml'
if not config_file.exists():
raise RuntimeError(
'Config file does not exist. '
'Unable to get repository information'
)
with config_file.open() as f:
config = toml.loads(f.read())
if (
'repositories' not in config
or repository_name not in config['repositories']
):
raise RuntimeError(
f'Repository {repository_name} is not defined'
)
url = config['repositories'][repository_name]['url']
username = None
password = None
auth_file = Path(CONFIG_DIR) / 'auth.toml'
if auth_file.exists():
with auth_file.open() as f:
auth_config = toml.loads(f.read())
if 'http-basic' in auth_config and repository_name in auth_config['http-basic']:
config = auth_config['http-basic'][repository_name]
username = config.get('username')
password = config.get('password')
# Requesting missing credentials
if not username:
username = self._io.ask('Username:')
if not password:
password = self._io.ask_hidden('Password:')
session = requests.session()
session.auth = (username, password)
session.headers['User-Agent'] = self._make_user_agent_string()
for scheme in ('http://', 'https://'):
session.mount(scheme, self._make_adapter_with_retries())
# TODO: handle certificates
try:
self.upload(session, url)
finally:
session.close()
def register(self, session, url):
"""
Register a package to a repository.
"""
dist = self._poetry.file.parent / 'dist'
file = dist / f'{self._package.name}-{self._package.version}.tar.gz'
if not file.exists():
raise RuntimeError(
'"{0}" does not exist on the file system.'.format(file.name)
)
data = self.post_data(file)
data.update({
":action": "submit",
"protocol_version": "1",
})
data_to_send = self._convert_data_to_list_of_tuples(data)
encoder = MultipartEncoder(data_to_send)
resp = session.post(
url,
data=encoder,
allow_redirects=False,
headers={'Content-Type': encoder.content_type},
)
return resp
def upload(self, session, url):
"""
Upload packages for the current project.
"""
try:
self._upload(session, url)
except HTTPError as e:
if (
e.response.status_code not in (403, 400)
or e.response.status_code == 400 and 'was ever registered' not in e.response.text
):
raise
# It may be the first time we publish the package
# We'll try to register it and go from there
try:
self.register(session, url)
except HTTPError:
raise
def post_data(self, file):
meta = Metadata.from_package(self._package)
file_type = self._get_type(file)
blake2_256_hash = hashlib.blake2b(digest_size=256 // 8)
md5_hash = hashlib.md5()
sha2_hash = hashlib.sha256()
with file.open('rb') as fp:
for content in iter(lambda: fp.read(io.DEFAULT_BUFFER_SIZE), b''):
md5_hash.update(content)
sha2_hash.update(content)
blake2_256_hash.update(content)
md5_digest = md5_hash.hexdigest()
sha2_digest = sha2_hash.hexdigest()
blake2_256_digest = blake2_256_hash.hexdigest()
if file_type == 'bdist_wheel':
wheel_info = wheel_file_re.match(file.name)
py_version = wheel_info.group("pyver")
else:
py_version = None
return {
# identify release
"name": meta.name,
"version": meta.version,
# file content
"filetype": file_type,
"pyversion": py_version,
# additional meta-data
"metadata_version": meta.metadata_version,
"summary": meta.summary,
"home_page": meta.home_page,
"author": meta.author,
"author_email": meta.author_email,
"maintainer": meta.maintainer,
"maintainer_email": meta.maintainer_email,
"license": meta.license,
"description": meta.description,
"keywords": meta.keywords,
"platform": meta.platforms,
"classifiers": meta.classifiers,
"download_url": meta.download_url,
"supported_platform": meta.supported_platforms,
"comment": None,
"md5_digest": md5_digest,
"sha256_digest": sha2_digest,
"blake2_256_digest": blake2_256_digest,
# PEP 314
"provides": meta.provides,
"requires": meta.requires,
"obsoletes": meta.obsoletes,
# Metadata 1.2
"project_urls": meta.project_urls,
"provides_dist": meta.provides_dist,
"obsoletes_dist": meta.obsoletes_dist,
"requires_dist": meta.requires_dist,
"requires_external": meta.requires_external,
"requires_python": meta.requires_python,
}
def _upload(self, session, url):
dist = self._poetry.file.parent / 'dist'
packages = dist.glob(f'{self._package.name}-{self._package.version}*')
files = (i for i in packages if i.suffix != '.asc')
for file in files:
# TODO; Check existence
resp = self._upload_file(session, url, file)
# Bug 92. If we get a redirect we should abort because something seems
# funky. The behaviour is not well defined and redirects being issued
# by PyPI should never happen in reality. This should catch malicious
# redirects as well.
if resp.is_redirect:
raise RuntimeError(
('"{0}" attempted to redirect to "{1}" during upload.'
' Aborting...').format(url, resp.headers["location"]))
resp.raise_for_status()
def _upload_file(self, session, url, file):
data = self.post_data(file)
data.update({
# action
":action": "file_upload",
"protocol_version": "1",
})
data_to_send = self._convert_data_to_list_of_tuples(data)
with file.open('rb') as fp:
data_to_send.append((
"content",
(file.name, fp, "application/octet-stream"),
))
encoder = MultipartEncoder(data_to_send)
bar = self._io.create_progress_bar(encoder.len)
bar.set_format(
" - Uploading <info>{0}</> <comment>%percent%%</>".format(
file.name
)
)
monitor = MultipartEncoderMonitor(
encoder, lambda monitor: bar.set_progress(monitor.bytes_read)
)
bar.start()
resp = session.post(
url,
data=monitor,
allow_redirects=False,
headers={'Content-Type': monitor.content_type}
)
if resp.ok:
bar.finish()
self._io.writeln('')
else:
self._io.overwrite('')
return resp
def _convert_data_to_list_of_tuples(self, data):
data_to_send = []
for key, value in data.items():
if (key in KEYWORDS_TO_NOT_FLATTEN or
not isinstance(value, (list, tuple))):
data_to_send.append((key, value))
else:
for item in value:
data_to_send.append((key, item))
return data_to_send
def _get_type(self, file):
exts = file.suffixes
if exts[-1] == '.whl':
return 'bdist_wheel'
elif len(exts) >= 2 and ''.join(exts[-2:]) == '.tar.gz':
return 'sdist'
raise ValueError(
f'Unknown distribution format {"".join(exts)}'
)
@staticmethod
def _make_adapter_with_retries():
retry = util.Retry(
connect=5,
total=10,
method_whitelist=['GET'],
status_forcelist=[500, 501, 502, 503],
)
return adapters.HTTPAdapter(max_retries=retry)
@staticmethod
def _make_user_agent_string():
return user_agent(
'twine', __version__,
)
poetry/packages/package.py
import re import re
from typing import Union from typing import Union
from poetry.semver.constraints import Constraint
from poetry.semver.helpers import parse_stability from poetry.semver.helpers import parse_stability
from poetry.semver.version_parser import VersionParser from poetry.semver.version_parser import VersionParser
from poetry.version import parse as parse_version from poetry.version import parse as parse_version
...@@ -13,6 +14,13 @@ AUTHOR_REGEX = re.compile('(?u)^(?P<name>[- .,\w\d\'’"()]+) <(?P<email>.+?)>$' ...@@ -13,6 +14,13 @@ AUTHOR_REGEX = re.compile('(?u)^(?P<name>[- .,\w\d\'’"()]+) <(?P<email>.+?)>$'
class Package: class Package:
AVAILABLE_PYTHONS = {
'2',
'2.7',
'3',
'3.4', '3.5', '3.6', '3.7'
}
supported_link_types = { supported_link_types = {
'require': { 'require': {
'description': 'requires', 'description': 'requires',
...@@ -174,6 +182,30 @@ class Package: ...@@ -174,6 +182,30 @@ class Package:
def platform_constraint(self): def platform_constraint(self):
return self._platform_constraint return self._platform_constraint
@property
def classifiers(self):
classifiers = []
# Automatically set python classifiers
parser = VersionParser()
if self.python_versions == '*':
python_constraint = parser.parse_constraints('~2.7 || ^3.4')
else:
python_constraint = self.python_constraint
for version in sorted(self.AVAILABLE_PYTHONS):
if len(version) == 1:
constraint = parser.parse_constraints(version + '.*')
else:
constraint = Constraint('=', version)
if python_constraint.matches(constraint):
classifiers.append(
f'Programming Language :: Python :: {version}'
)
return classifiers
def is_dev(self): def is_dev(self):
return self._dev return self._dev
......
pyproject.toml
...@@ -7,7 +7,7 @@ authors = [ ...@@ -7,7 +7,7 @@ authors = [
] ]
license = "MIT" license = "MIT"
readme = "README.md" readme = "README.rst"
homepage = "https://poetry.eustace.io/" homepage = "https://poetry.eustace.io/"
repository = "https://github.com/sdispater/poet" repository = "https://github.com/sdispater/poet"
...@@ -23,7 +23,6 @@ requests = "^2.18" ...@@ -23,7 +23,6 @@ requests = "^2.18"
toml = "^0.9" toml = "^0.9"
cachy = "^0.1.0" cachy = "^0.1.0"
pip-tools = "^1.11" pip-tools = "^1.11"
twine = "^1.10"
requests-toolbelt = "^0.8.0" requests-toolbelt = "^0.8.0"
[tool.poetry.dev-dependencies] [tool.poetry.dev-dependencies]
......
setup.py
...@@ -38,7 +38,6 @@ kwargs = dict( ...@@ -38,7 +38,6 @@ kwargs = dict(
'toml>=0.9.4,<0.10.0', 'toml>=0.9.4,<0.10.0',
'cachy>=0.1.0,<0.2.0', 'cachy>=0.1.0,<0.2.0',
'pip-tools>=1.11.0,<2.0.0', 'pip-tools>=1.11.0,<2.0.0',
'twine>=^1.10.0,<2.0.0',
'requests-toolbelt>=^0.8.0,<0.9.0', 'requests-toolbelt>=^0.8.0,<0.9.0',
], ],
include_package_data=True, include_package_data=True,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment