src_prepare/euscan-ng
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Use JSON api for PyPi

Browse Source 
* "The XML-RPC API will be deprecated in the future. Use of this API is
  not recommended, and existing consumers of the API should migrate to
  the RSS and/or JSON APIs instead."
* "As a result, this API has a very restrictive rate limit and it may be
  necessary to pause between successive requests." As such this also
  gets around this issue for euscan.

https://warehouse.pypa.io/api-reference/xml-rpc.html

Signed-off-by: Alfred Wingate <parona@protonmail.com>
This commit is contained in:
Alfred Wingate 2024-03-02 04:31:37 +02:00
parent 9465c14342
commit aad99f71fe
No known key found for this signature in database
GPG Key ID: A12750536B5E7010
2 changed files with 28 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
pyproject.toml
View File

@ -16,7 +16,8 @@ description = "Ebuild upstream scan utility."
license = {text = "GPL-2.0"} license = {text = "GPL-2.0"}
dependencies = [ dependencies = [
"portage", "portage",
"beautifulsoup4>=4.8.2" "beautifulsoup4>=4.8.2",
"packaging"
] ]
dynamic = ["version"] dynamic = ["version"]

37
src/euscan/handlers/pypi.py
View File

@ -1,11 +1,13 @@
# Copyright 2011 Corentin Chary <corentin.chary@gmail.com> # Copyright 2011 Corentin Chary <corentin.chary@gmail.com>
# Copyright 2020-2023 src_prepare group # Copyright 2020-2024 src_prepare group
# Distributed under the terms of the GNU General Public License v2 # Distributed under the terms of the GNU General Public License v2
import json
import re import re
import xmlrpc.client import urllib.error
import portage import portage
from packaging.version import parse
from euscan import helpers, mangling, output from euscan import helpers, mangling, output
@ -29,7 +31,7 @@ def guess_package(cp, url):
def scan_url(pkg, url, options): def scan_url(pkg, url, options):
"http://wiki.python.org/moin/PyPiXmlRpc" "https://peps.python.org/pep-0691/"
package = guess_package(pkg.cpv, url) package = guess_package(pkg.cpv, url)
return scan_pkg(pkg, {"data": package}) return scan_pkg(pkg, {"data": package})
@ -38,15 +40,23 @@ def scan_url(pkg, url, options):
def scan_pkg(pkg, options): def scan_pkg(pkg, options):
package = options["data"] package = options["data"]
output.einfo("Using PyPi XMLRPC: " + package) output.einfo("Using PyPi JSON API: " + package)
client = xmlrpc.client.ServerProxy("https://pypi.python.org/pypi") try:
versions = client.package_releases(package) fp = helpers.urlopen(f"https://pypi.org/pypi/{package}/json/")
except urllib.error.URLError:
return []
except OSError:
return []
if not versions: if not fp:
return versions return []
versions.reverse() data = json.loads(fp.read())
versions = list(data["releases"].keys())
versions.sort(key=parse, reverse=True)
cp, ver, rev = portage.pkgsplit(pkg.cpv) cp, ver, rev = portage.pkgsplit(pkg.cpv)
@ -55,7 +65,12 @@ def scan_pkg(pkg, options):
pv = mangling.mangle_version(up_pv, options) pv = mangling.mangle_version(up_pv, options)
if helpers.version_filtered(cp, ver, pv): if helpers.version_filtered(cp, ver, pv):
continue continue
urls = client.release_urls(package, up_pv) urls = " ".join(
urls = " ".join([mangling.mangle_url(infos["url"], options) for infos in urls]) [
mangling.mangle_url(file["url"], options)
for file in data["releases"][up_pv]
if file["packagetype"] == "sdist"
]
)
ret.append((urls, pv, HANDLER_NAME, CONFIDENCE)) ret.append((urls, pv, HANDLER_NAME, CONFIDENCE))
return ret return ret