Make index urls a set to avoid exporting duplicates to requirements.txt (#1610)

* Make index urls a set to avoid exporting duplicates to requirements.txt

Fixes #1434

* Add test for deduplication of sources in exporter

* Holy test names, blackman.

* Sort indexes for consistent output with multiple extra indexes

poetry/utils/exporter.py

@@ -54,7 +54,7 @@ class Exporter(object):
         extras=None,
         extras=None,
         with_credentials=False,
         with_credentials=False,
     ):  # type: (Path, Union[IO, str], bool, bool, bool) -> None
     ):  # type: (Path, Union[IO, str], bool, bool, bool) -> None
-        indexes = []
+        indexes = set()
         content = ""
         content = ""
         packages = self._poetry.locker.locked_repository(dev).packages
         packages = self._poetry.locker.locked_repository(dev).packages
 
 
@@ -108,7 +108,7 @@ class Exporter(object):
                 package.source_type not in {"git", "directory", "file", "url"}
                 package.source_type not in {"git", "directory", "file", "url"}
                 and package.source_url
                 and package.source_url
             ):
             ):
-                indexes.append(package.source_url)
+                indexes.add(package.source_url)
 
 
             if package.files and with_hashes:
             if package.files and with_hashes:
                 hashes = []
                 hashes = []
@@ -134,10 +134,9 @@ class Exporter(object):
             content += line
             content += line
 
 
         if indexes:
         if indexes:
-            # If we have extra indexes, we add them to the begin
-            # of the output
+            # If we have extra indexes, we add them to the beginning of the output
             indexes_header = ""
             indexes_header = ""
-            for index in indexes:
+            for index in sorted(indexes):
                 repository = [
                 repository = [
                     r
                     r
                     for r in self._poetry.pool.repositories
                     for r in self._poetry.pool.repositories

tests/utils/test_exporter.py

@@ -707,6 +707,86 @@ foo==1.2.3 \\
     assert expected == content
     assert expected == content
 
 
 
 
+def test_exporter_exports_requirements_txt_with_legacy_packages_and_duplicate_sources(
+    tmp_dir, poetry
+):
+    poetry.pool.add_repository(
+        LegacyRepository(
+            "custom",
+            "https://example.com/simple",
+            auth=Auth("https://example.com/simple", "foo", "bar"),
+        )
+    )
+    poetry.pool.add_repository(LegacyRepository("custom", "https://foobaz.com/simple",))
+    poetry.locker.mock_lock_data(
+        {
+            "package": [
+                {
+                    "name": "foo",
+                    "version": "1.2.3",
+                    "category": "main",
+                    "optional": False,
+                    "python-versions": "*",
+                    "source": {
+                        "type": "legacy",
+                        "url": "https://example.com/simple",
+                        "reference": "",
+                    },
+                },
+                {
+                    "name": "bar",
+                    "version": "4.5.6",
+                    "category": "dev",
+                    "optional": False,
+                    "python-versions": "*",
+                    "source": {
+                        "type": "legacy",
+                        "url": "https://example.com/simple",
+                        "reference": "",
+                    },
+                },
+                {
+                    "name": "baz",
+                    "version": "7.8.9",
+                    "category": "dev",
+                    "optional": False,
+                    "python-versions": "*",
+                    "source": {
+                        "type": "legacy",
+                        "url": "https://foobaz.com/simple",
+                        "reference": "",
+                    },
+                },
+            ],
+            "metadata": {
+                "python-versions": "*",
+                "content-hash": "123456789",
+                "hashes": {"foo": ["12345"], "bar": ["67890"], "baz": ["24680"]},
+            },
+        }
+    )
+    exporter = Exporter(poetry)
+
+    exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt", dev=True)
+
+    with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f:
+        content = f.read()
+
+    expected = """\
+--extra-index-url https://example.com/simple
+--extra-index-url https://foobaz.com/simple
+
+bar==4.5.6 \\
+    --hash=sha256:67890
+baz==7.8.9 \\
+    --hash=sha256:24680
+foo==1.2.3 \\
+    --hash=sha256:12345
+"""
+
+    assert expected == content
+
+
 def test_exporter_exports_requirements_txt_with_legacy_packages_and_credentials(
 def test_exporter_exports_requirements_txt_with_legacy_packages_and_credentials(
     tmp_dir, poetry, config
     tmp_dir, poetry, config
 ):
 ):