Compare commits

...

9 Commits

Author SHA1 Message Date
Alfred Wingate 5b9d44fee1
TODO: weird docs handling
Signed-off-by: Alfred Wingate <parona@protonmail.com>
2024-03-03 20:58:58 +02:00
Alfred Wingate d8d1767766
TODO: debian remote-id exists
Signed-off-by: Alfred Wingate <parona@protonmail.com>
2024-03-03 20:58:58 +02:00
Alfred Wingate fbd7a4e139
handlers/github: remove
* Mirror removed and api has very strict ratelimits making impractical
  to use.
* https://gitweb.gentoo.org/repo/gentoo.git/commit/?id=f119d00dab0c3bd087faab36f1a44734772a9d75

Signed-off-by: Alfred Wingate <parona@protonmail.com>
2024-03-03 20:58:58 +02:00
Alfred Wingate a7ff66ae04
handlers/pypi: stop using mirrors
Signed-off-by: Alfred Wingate <parona@protonmail.com>
2024-03-03 20:58:58 +02:00
Alfred Wingate 5da26b0719
handlers/rubygems: stop using mirrors
* https://gitweb.gentoo.org/repo/gentoo.git/commit/?id=65850a10f84e1b7a2cdf55392fa1d1f0717193c1

Signed-off-by: Alfred Wingate <parona@protonmail.com>
2024-03-03 20:58:58 +02:00
Alfred Wingate 656f8e155e
handlers/google_code: dead
* https://bugs.gentoo.org/544092

Signed-off-by: Alfred Wingate <parona@protonmail.com>
2024-03-03 20:58:57 +02:00
Alfred Wingate 294dcc2a9c
handlers/freecode: never shouldve been used in ebuilds
* https://bugs.gentoo.org/637970

Signed-off-by: Alfred Wingate <parona@protonmail.com>
2024-03-03 20:58:57 +02:00
Alfred Wingate c628edc26b
handlers/berlios: obselete
* mirror removed in https://gitweb.gentoo.org/repo/gentoo.git/commit/?id=2b72b0462bea5b34bbe4d767ccc44866df81515e
* Rest of the berlios urls use sourceforge now.

Signed-off-by: Alfred Wingate <parona@protonmail.com>
2024-03-03 19:36:20 +02:00
Alfred Wingate 61cbb8e3f9
pre-commit: autoupdate versions
Signed-off-by: Alfred Wingate <parona@protonmail.com>
2024-03-03 19:11:43 +02:00
9 changed files with 12 additions and 241 deletions

View File

@ -1,15 +1,15 @@
repos:
- repo: https://github.com/psf/black
rev: 23.11.0
rev: 24.2.0
hooks:
- id: black
- repo: https://github.com/PyCQA/isort
rev: 5.12.0
rev: 5.13.2
hooks:
- id: isort
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.1.4
rev: v0.3.0
hooks:
- id: ruff

10
TODO
View File

@ -42,11 +42,7 @@ euscan
- remote-id type deb repository:
-- find out how to get download url (not sure it's possible)
### remote-id
- Propose new remote-id: deb
e.g.: <remote-id type="deb">
http://mysite.com/deb/dists/stable/main/binary-i386/Packages
</remote-id>
- Propose new remote-id: freecode
e.g.: <remote-id type="freecode">projectname</remote-id>
### bugs or unwanted behavior
- Parsing docs and accepting 404's
-- net-analyzer/sensu

View File

@ -51,7 +51,7 @@ BLACKLIST_PACKAGES = [
]
SCANDIR_BLACKLIST_URLS = [
"mirror://rubygems/(.*)", # Not browsable
"https://rubygems.org/(.*)", # Not browsable
"mirror://gentoo/(.*)", # Directory too big
"https://dev.gentoo.org/(.*)", # There shouldn't be releases here
# Waste of time to go through

View File

@ -1,59 +0,0 @@
# Copyright 2011 Corentin Chary <corentin.chary@gmail.com>
# Copyright 2020-2023 src_prepare group
# Distributed under the terms of the GNU General Public License v2
import re
import urllib.error
import urllib.parse
import urllib.request
import portage
from euscan import output
from euscan.handlers.url import process_scan as url_scan
from euscan.helpers import regex_from_template
HANDLER_NAME = "berlios"
CONFIDENCE = 90
PRIORITY = 90
berlios_regex = r"mirror://berlios/([^/]+)/([^/]+)"
def can_handle(pkg, url=None):
if not url:
return False
cp, ver, rev = portage.pkgsplit(pkg.cpv)
if ver not in url:
return False
return re.search(berlios_regex, url)
def scan_url(pkg, url, options):
output.einfo("Using BerliOS handler")
cp, ver, rev = portage.pkgsplit(pkg.cpv)
project, filename = re.search(berlios_regex, url).groups()
project_page = "http://developer.berlios.de/projects/%s" % project
content = urllib.request.urlopen(project_page).read()
project_id = re.search(r"/project/filelist.php\?group_id=(\d+)", content).group(1)
base_url = (
"http://developer.berlios.de/project/filelist.php?group_id=%s" % project_id
)
file_pattern = regex_from_template(filename.replace(ver, "${PV}"))
result = url_scan(pkg, base_url, file_pattern)
ret = []
for found_url, pv, _, _ in result:
found_url = found_url.replace("prdownload", "download")
ret.append((found_url, pv, HANDLER_NAME, CONFIDENCE))
return ret

View File

@ -1,53 +0,0 @@
# Copyright 2011 Corentin Chary <corentin.chary@gmail.com>
# Copyright 2020-2023 src_prepare group
# Distributed under the terms of the GNU General Public License v2
import re
import urllib.error
import urllib.parse
import urllib.request
import portage
from euscan import helpers, mangling, output
HANDLER_NAME = "freecode"
CONFIDENCE = 100
PRIORITY = 90
def can_handle(pkg, url=None):
return False
def scan_pkg(pkg, options):
cp, ver, rev = portage.pkgsplit(pkg.cpv)
package = options["data"].strip()
output.einfo("Using FreeCode handler: " + package)
fp = urllib.request.urlopen("http://freecode.com/projects/%s/releases" % package)
content = str(fp.read())
result = re.findall(
r'<a href="/projects/%s/releases/(\d+)">([^<]+)</a>' % package, content
)
ret = []
for release_id, up_pv in result:
pv = mangling.mangle_version(up_pv, options)
if helpers.version_filtered(cp, ver, pv):
continue
fp = urllib.request.urlopen(
f"http://freecode.com/projects/{package}/releases/{release_id}"
)
content = str(fp.read())
download_page = re.findall(r'<a href="(/urls/[^"]+)"', content)[0]
fp = urllib.request.urlopen("http://freecode.com%s" % download_page)
content = str(fp.read())
url = re.findall(
r'In case it doesn\'t, click here: <a href="([^"]+)"', content
)[0]
ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
return ret

View File

@ -1,66 +0,0 @@
# Copyright 2011 Corentin Chary <corentin.chary@gmail.com>
# Copyright 2020-2023 src_prepare group
# Distributed under the terms of the GNU General Public License v2
import json
import re
import urllib.error
import urllib.parse
import urllib.request
import portage
from euscan import helpers, mangling, output
HANDLER_NAME = "github"
CONFIDENCE = 100
PRIORITY = 90
def can_handle(pkg, url=None):
return url and url.startswith("mirror://github/")
def guess_package(cp, url):
match = re.search("^mirror://github/(.*?)/(.*?)/(.*)$", url)
assert match
return (match.group(1), match.group(2), match.group(3))
def scan_url(pkg, url, options):
"http://developer.github.com/v3/repos/downloads/"
user, project, filename = guess_package(pkg.cpv, url)
# find out where version is expected to be found
cp, ver, rev = portage.pkgsplit(pkg.cpv)
if ver not in filename:
return
# now create a filename-matching regexp
# XXX: supposedly replace first with (?P<foo>...)
# and remaining ones with (?P=foo)
fnre = re.compile("^%s$" % re.escape(filename).replace(re.escape(ver), "(.*?)"))
output.einfo(
f"Using github API for: project={project} user={user} filename={filename}"
)
dlreq = urllib.request.urlopen(
f"https://api.github.com/repos/{user}/{project}/downloads"
)
dls = json.load(dlreq)
ret = []
for dl in dls:
m = fnre.match(dl["name"])
if m:
pv = mangling.mangle_version(m.group(1), options)
if helpers.version_filtered(cp, ver, pv):
continue
url = mangling.mangle_url(dl["html_url"], options)
ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
return ret

View File

@ -1,47 +0,0 @@
# Copyright 2011 Corentin Chary <corentin.chary@gmail.com>
# Copyright 2020-2023 src_prepare group
# Distributed under the terms of the GNU General Public License v2
import re
import portage
from euscan import output
from euscan.handlers.url import process_scan as url_scan
from euscan.helpers import regex_from_template
HANDLER_NAME = "google-code"
CONFIDENCE = 90
PRIORITY = 90
package_name_regex = r"http://(.+).googlecode.com/files/.+"
def can_handle(pkg, url=None):
if not url:
return False
cp, ver, rev = portage.pkgsplit(pkg.cpv)
if ver not in url:
return False
return re.match(package_name_regex, url)
def scan_url(pkg, url, options):
output.einfo("Using Google Code handler")
cp, ver, rev = portage.pkgsplit(pkg.cpv)
package_name = re.match(package_name_regex, url).group(1)
base_url = "http://code.google.com/p/%s/downloads/list" % package_name
file_pattern = regex_from_template(url.split("/")[-1].replace(ver, "${PV}"))
result = url_scan(pkg, base_url, file_pattern)
ret = []
for url, pv, _, _ in result:
ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
return ret

View File

@ -17,11 +17,11 @@ PRIORITY = 90
def can_handle(pkg, url=None):
return url and url.startswith("mirror://pypi/")
return url and url.startswith("https://files.pythonhosted.org/packages/source/p/")
def guess_package(cp, url):
match = re.search(r"mirror://pypi/\w+/(.*)/.*", url)
match = re.search(r"https://files.pythonhosted.org/packages/source/p/(.*)/.*", url)
if match:
return match.group(1)

View File

@ -1,5 +1,5 @@
# Copyright 2011 Corentin Chary <corentin.chary@gmail.com>
# Copyright 2020-2023 src_prepare group
# Copyright 2020-2024 src_prepare group
# Distributed under the terms of the GNU General Public License v2
import json
@ -18,11 +18,11 @@ PRIORITY = 90
def can_handle(pkg, url=None):
return url and url.startswith("mirror://rubygems/")
return url and url.startswith("https://rubygems.org/")
def guess_gem(cpv, url):
match = re.search("mirror://rubygems/(.*).gem", url)
match = re.search("https://rubygems.org/gems/(.*).gem", url)
if match:
cpv = "fake/%s" % match.group(1)