feat(probe-builder): Ability to crawl EulerOS kernel packages

probe_builder/__init__.py

@@ -84,6 +84,7 @@ def get_kernels(self, _workspace, packages, _download_config, _crawler_filter):
     'CentOS': CrawlDistro('centos', 'centos', 'CentOS'),
     'CentOSStream': CrawlDistro('centosstream', 'centos', 'CentOSStream'),
     'Debian': CrawlDistro('debian', 'debian', 'Debian'),
+    'EulerOS': CrawlDistro('euleros', 'centos', 'EulerOS'),
     'Fedora': CrawlDistro('fedora', 'centos', 'Fedora'),
     'Flatcar': CrawlDistro('flatcar', 'flatcar', 'Flatcar'),
     'Oracle6': CrawlDistro('oracle6', 'oracle', 'Oracle6'),

probe_builder/kernel_crawler/__init__.py

@@ -2,6 +2,7 @@
 from .almalinux import AlmaLinuxMirror
 from .amazonlinux import AmazonLinux1Mirror, AmazonLinux2Mirror, AmazonLinux2022Mirror
 from .centos import CentosMirror, CentosStreamMirror
+from .euleros import EulerOSMirror
 from .fedora import FedoraMirror
 from .oracle import Oracle6Mirror, Oracle7Mirror, Oracle8Mirror, Oracle9Mirror
 from .photon_os import PhotonOsMirror
@@ -20,6 +21,7 @@
     'AmazonLinux2022': AmazonLinux2022Mirror,
     'CentOS': CentosMirror,
     'CentOSStream': CentosStreamMirror,
+    'EulerOS': EulerOSMirror,
     'Fedora': FedoraMirror,
     'Oracle6': Oracle6Mirror,
     'Oracle7': Oracle7Mirror,

probe_builder/kernel_crawler/euleros.py

@@ -0,0 +1,19 @@
+from . import repo
+from . import rpm
+
+def v2(ver):
+    return ver.startswith('2')
+
+class EulerOSMirror(repo.Distro):
+    def get_mirrors(self, crawler_filter):
+        mirrors = [
+            # EulerOS 2
+            # Lifecycle:
+            # https://developer.huaweicloud.com/intl/en-us/euleros/lifecycle-management.html
+            # Mirror list:
+            # http://mirrors.huaweicloud.com/euler/
+            rpm.RpmMirror('http://mirrors.huaweicloud.com/euler/', 'os/{}/'.format(crawler_filter.machine), v2),
+            rpm.RpmMirror('http://mirrors.huaweicloud.com/euler/', 'updates/{}/'.format(crawler_filter.machine), v2),
+
+        ]
+        return mirrors

probe_builder/kernel_crawler/rpm.py

@@ -118,7 +118,10 @@ def list_drel_repos(self, crawler_filter):
         dists.raise_for_status()
         dists = dists.content
         doc = html.fromstring(dists, self.base_url)
-        dists = doc.xpath('/html/body//a[not(@href="../")]/@href')
+        # Huawei Cloud loves do things differently
+        # Their page uses a table ouside body tags
+        # this additional filter catch it
+        dists = doc.xpath('/html/body//a[not(@href="../")]/@href | //table/tbody//a[not(@href="../")]/@href')
 
         fdists = [dist for dist in dists
                 if dist.endswith('/')