Improve PyPI search by dropping the XML RPC API (#1530)

poetry/console/commands/search.py

 from cleo import argument
-from cleo import option
 
 from .command import Command
 
@@ -10,16 +9,11 @@ class SearchCommand(Command):
     description = "Searches for packages on remote repositories."
 
     arguments = [argument("tokens", "The tokens to search for.", multiple=True)]
-    options = [option("only-name", "N", "Search only by name.")]
 
     def handle(self):
         from poetry.repositories.pypi_repository import PyPiRepository
 
-        flags = PyPiRepository.SEARCH_FULLTEXT
-        if self.option("only-name"):
-            flags = PyPiRepository.SEARCH_NAME
-
-        results = PyPiRepository().search(self.argument("tokens"), flags)
+        results = PyPiRepository().search(self.argument("tokens"))
 
         for result in results:
             self.line("")

poetry/repositories/base_repository.py

 class BaseRepository(object):
-
-    SEARCH_FULLTEXT = 0
-    SEARCH_NAME = 1
-
     def __init__(self):
         self._packages = []
 
@@ -21,5 +17,5 @@ class BaseRepository(object):
     ):
         raise NotImplementedError()
 
-    def search(self, query, mode=SEARCH_FULLTEXT):
+    def search(self, query):
         raise NotImplementedError()

poetry/repositories/pool.py

@@ -151,7 +151,7 @@ class Pool(BaseRepository):
 
         return packages
 
-    def search(self, query, mode=BaseRepository.SEARCH_FULLTEXT):
+    def search(self, query):
         from .legacy_repository import LegacyRepository
 
         results = []
@@ -159,6 +159,6 @@ class Pool(BaseRepository):
             if isinstance(repository, LegacyRepository):
                 continue
 
-            results += repository.search(query, mode=mode)
+            results += repository.search(query)
 
         return results

poetry/repositories/pypi_repository.py

 import logging
 import os
 
-
 from collections import defaultdict
 from typing import Dict
 from typing import List
@@ -12,14 +11,10 @@ try:
 except ImportError:
     import urlparse
 
-try:
-    from xmlrpc.client import ServerProxy
-except ImportError:
-    from xmlrpclib import ServerProxy
-
 from cachecontrol import CacheControl
 from cachecontrol.caches.file_cache import FileCache
 from cachy import CacheManager
+from html5lib.html5parser import parse
 from requests import get
 from requests import session
 
@@ -33,11 +28,9 @@ from poetry.semver import VersionRange
 from poetry.semver.exceptions import ParseVersionError
 from poetry.utils._compat import Path
 from poetry.utils._compat import to_str
-from poetry.utils.helpers import parse_requires
 from poetry.utils.helpers import temporary_directory
 from poetry.utils.inspector import Inspector
 from poetry.utils.patterns import wheel_file_re
-from poetry.utils.setup_reader import SetupReader
 from poetry.version.markers import InvalidMarker
 from poetry.version.markers import parse_marker
 
@@ -222,26 +215,32 @@ class PyPiRepository(Repository):
 
         return package
 
-    def search(self, query, mode=0):
+    def search(self, query):
         results = []
 
-        search = {"name": query}
+        search = {"q": query}
 
-        if mode == self.SEARCH_FULLTEXT:
-            search["summary"] = query
+        response = session().get(self._url + "search", params=search)
+        content = parse(response.content, namespaceHTMLElements=False)
+        for result in content.findall(".//*[@class='package-snippet']"):
+            name = result.find("h3/*[@class='package-snippet__name']").text
+            version = result.find("h3/*[@class='package-snippet__version']").text
 
-        client = ServerProxy("https://pypi.python.org/pypi")
-        hits = client.search(search, "or")
+            if not name or not version:
+                continue
+
+            description = result.find("p[@class='package-snippet__description']").text
+            if not description:
+                description = ""
 
-        for hit in hits:
             try:
-                result = Package(hit["name"], hit["version"], hit["version"])
-                result.description = to_str(hit["summary"])
+                result = Package(name, version, description)
+                result.description = to_str(description.strip())
                 results.append(result)
             except ParseVersionError:
                 self._log(
                     'Unable to parse version "{}" for the {} package, skipping'.format(
-                        hit["version"], hit["name"]
+                        version, name
                     ),
                     level="debug",
                 )

poetry/repositories/repository.py

@@ -115,7 +115,7 @@ class Repository(BaseRepository):
         if index is not None:
             del self._packages[index]
 
-    def search(self, query, mode=0):
+    def search(self, query):
         results = []
 
         for package in self.packages:

tests/console/commands/test_search.py

+from cleo.testers import CommandTester
+
+from poetry.utils._compat import Path
+
+
+TESTS_DIRECTORY = Path(__file__).parent.parent.parent
+FIXTURES_DIRECTORY = (
+    TESTS_DIRECTORY / "repositories" / "fixtures" / "pypi.org" / "search"
+)
+
+
+def test_search(app, http):
+    with FIXTURES_DIRECTORY.joinpath("search.html").open(encoding="utf-8") as f:
+        search_results = f.read()
+
+    http.register_uri("GET", "https://pypi.org/search", search_results)
+
+    command = app.find("search")
+    tester = CommandTester(command)
+
+    tester.execute("sqlalchemy")
+
+    expected = """
+sqlalchemy (1.3.10)
+ Database Abstraction Library
+
+sqlalchemy-dao (1.3.1)
+ Simple wrapper for sqlalchemy.
+
+graphene-sqlalchemy (2.2.2)
+ Graphene SQLAlchemy integration
+
+sqlalchemy-utcdatetime (1.0.4)
+ Convert to/from timezone aware datetimes when storing in a DBMS
+
+paginate-sqlalchemy (0.3.0)
+ Extension to paginate.Page that supports SQLAlchemy queries
+
+sqlalchemy-audit (0.1.0)
+ sqlalchemy-audit provides an easy way to set up revision tracking for your data.
+
+transmogrify.sqlalchemy (1.0.2)
+ Feed data from SQLAlchemy into a transmogrifier pipeline
+
+sqlalchemy-schemadisplay (1.3)
+ Turn SQLAlchemy DB Model into a graph
+
+sqlalchemy-traversal (0.5.2)
+ UNKNOWN
+
+sqlalchemy-filters (0.10.0)
+ A library to filter SQLAlchemy queries.
+
+sqlalchemy-wrap (2.1.7)
+ Python wrapper for the CircleCI API
+
+sqlalchemy-nav (0.0.2)
+ SQLAlchemy-Nav provides SQLAlchemy Mixins for creating navigation bars compatible with Bootstrap
+
+sqlalchemy-repr (0.0.1)
+ Automatically generates pretty repr of a SQLAlchemy model.
+
+sqlalchemy-diff (0.1.3)
+ Compare two database schemas using sqlalchemy.
+
+sqlalchemy-equivalence (0.1.1)
+ Provides natural equivalence support for SQLAlchemy declarative models.
+
+broadway-sqlalchemy (0.0.1)
+ A broadway extension wrapping Flask-SQLAlchemy
+
+jsonql-sqlalchemy (1.0.1)
+ Simple JSON-Based CRUD Query Language for SQLAlchemy
+
+sqlalchemy-plus (0.2.0)
+ Create Views and Materialized Views with SqlAlchemy
+
+cherrypy-sqlalchemy (0.5.3)
+ Use SQLAlchemy with CherryPy
+
+sqlalchemy-sqlany (1.0.3)
+ SAP Sybase SQL Anywhere dialect for SQLAlchemy
+"""
+
+    assert expected == tester.io.fetch_output()