Merge pull request #75 from package-url/url2purl-updates

Url2purl updates
package-url · Feb 12, 2022 · 8f7c24e · 8f7c24e
2 parents 66bb781 + 5ebe515
commit 8f7c24e
Show file tree

Hide file tree

Showing 4 changed files with 75 additions and 37 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -1,6 +1,15 @@
 Changelog
 =========
 
+0.9.8 (2022-02-11)
+------------------
+
+- Do not create a generic PackageURL for URLs without a path in url2purl #72
+- Use project name as the Package name when creating generic sourceforge PackageURLs #74
+- Update PyPI route pattern in url2purl to handle different file name formats #76
+- Create generic PackageURL for code.google.com archive URLs #78
+- Capture more download types for bitbucket URLs
+
 0.9.7 (2022-02-07)
 ------------------
 

diff --git a/setup.cfg b/setup.cfg
@@ -1,6 +1,6 @@
 [metadata]
 name = packageurl-python
-version = 0.9.7
+version = 0.9.8
 license = MIT
 description = A purl aka. Package URL parser and builder
 long_description = file:README.rst

diff --git a/src/packageurl/contrib/url2purl.py b/src/packageurl/contrib/url2purl.py
@@ -102,14 +102,15 @@ def build_generic_purl(uri):
     if parsed_uri.scheme and parsed_uri.netloc and parsed_uri.path:
         # Get file name from `uri`
         uri_path_segments = get_path_segments(uri)
-        file_name = uri_path_segments[-1]
-        return PackageURL(
-            type='generic',
-            name=file_name,
-            qualifiers={
-                'download_url': uri
-            }
-        )
+        if uri_path_segments:
+            file_name = uri_path_segments[-1]
+            return PackageURL(
+                type='generic',
+                name=file_name,
+                qualifiers={
+                    'download_url': uri
+                }
+            )
 
 
 @purl_router.route('https?://registry.npmjs.*/.*',
@@ -262,9 +263,12 @@ def build_rubygems_purl(uri):
 
 
 # https://pypi.python.org/packages/source/a/anyjson/anyjson-0.3.3.tar.gz
+# https://pypi.python.org/packages/2.6/t/threadpool/threadpool-1.2.7-py2.6.egg
+# https://pypi.python.org/packages/any/s/setuptools/setuptools-0.6c11-1.src.rpm
+# https://files.pythonhosted.org/packages/84/d8/451842a5496844bb5c7634b231a2e4caf0d867d2e25f09b840d3b07f3d4b/multi_key_dict-2.0.win32.exe
 pypi_pattern = (
-    r"(?P<name>.+)-(?P<version>.+)"
-    r"\.(zip|tar.gz|tar.bz2|.tgz)$"
+    r"(?P<name>(\w\.?)+(-\w+)*)-(?P<version>.+)"
+    r"\.(zip|tar.gz|tar.bz2|tgz|egg|rpm|exe)$"
 )
 
 # This pattern can be found in the following locations:
@@ -341,18 +345,17 @@ def build_sourceforge_purl(uri):
     sourceforge_purl = purl_from_pattern('sourceforge', sourceforge_pattern, uri)
 
     if not sourceforge_purl:
-        # We create a more generic PackageURL from `uri` if `uri` doesn't fit
-        # `sourceforge_pattern`
-        uri_path_segments = get_path_segments(uri)
-        file_name = uri_path_segments[-1]
-        sourceforge_purl = PackageURL(
-            type='sourceforge',
-            name=file_name,
-            qualifiers={
-                'download_url': uri
-            }
-        )
-
+        # Get the project name from `uri` and use that as the Package name
+        # http://master.dl.sourceforge.net/project/aloyscore/aloyscore/0.1a1%2520stable/0.1a1_stable_AloysCore.zip
+        _, remaining_uri_path = uri.split('/project/') # http://master.dl.sourceforge.net, aloyscore/aloyscore/0.1a1%2520stable/0.1a1_stable_AloysCore.zip
+        if remaining_uri_path:
+            split_remaining_uri_path = remaining_uri_path.split('/') # aloyscore, aloyscore, 0.1a1%2520stable, 0.1a1_stable_AloysCore.zip
+            project_name = split_remaining_uri_path[0] # aloyscore
+            sourceforge_purl = PackageURL(
+                type='sourceforge',
+                name=project_name,
+                qualifiers={'download_url': uri}
+            )
     return sourceforge_purl
 
 
@@ -529,7 +532,7 @@ def build_bitbucket_purl(url):
 
     bitbucket_download_pattern = (
         r"https?://bitbucket.org/"
-        r"(?P<namespace>.+)/(?P<name>.+)/downloads/(?P<version>.+).(zip|tar.gz|tar.bz2|.tgz)"
+        r"(?P<namespace>.+)/(?P<name>.+)/downloads/(?P<version>.+).(zip|tar.gz|tar.bz2|.tgz|exe|msi)"
     )
     matches = re.search(bitbucket_download_pattern, url)
 
@@ -612,3 +615,19 @@ def build_gitlab_purl(url):
 )
 
 register_pattern('hackage', hackage_pattern)
+
+
+@purl_router.route('https?://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/.*')
+def build_generic_google_code_archive_purl(uri):
+    # https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/android-notifier/android-notifier-desktop-0.5.1-1.i386.rpm
+    _, remaining_uri = uri.split('https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/') # android-notifier/android-notifier-desktop-0.5.1-1.i386.rpm
+    if remaining_uri:
+        split_remaining_uri = remaining_uri.split("/") # android-notifier, android-notifier-desktop-0.5.1-1.i386.rpm
+        if split_remaining_uri:
+            name = split_remaining_uri[0] # android-notifier
+            return PackageURL(
+                type='generic',
+                namespace='code.google.com',
+                name=name,
+                qualifiers={'download_url': uri}
+            )
diff --git a/tests/contrib/data/url2purl.json b/tests/contrib/data/url2purl.json
@@ -131,6 +131,12 @@
   "https://pypi.python.org/packages/f6/ae/bbc6a204f33d9d57c798fb3857a072cd14b836792244eea4b446fdb674c6/pycryptodome-3.4.7-cp27-cp27m-win32.whl#md5=78b341de1cd686077745cd9e3a93d8d3": "pkg:pypi/[email protected]",
   "https://pypi.python.org/packages/bd/e8/ea44ba5357a0b4fd16e5fb60c355fc8722eae31b93d7597eec50f7c35a52/pycryptodome-3.4.7-cp27-cp27m-win_amd64.whl#md5=f20bb847322baf7ae24700e5cbb15e07": "pkg:pypi/[email protected]",
   "https://pypi.python.org/packages/1e/75/8005d086cac4cc41d3b320d338972c5e5c6a21f88472f21ac9d0e031d300/pyahocorasick-1.1.4.tar.bz2#md5=ad445b6648dc06e9040705ce1ccb4384": "pkg:pypi/[email protected]",
+  "https://pypi.python.org/packages/2.6/t/threadpool/threadpool-1.2.7-py2.6.egg": "pkg:pypi/[email protected]",
+  "https://pypi.python.org/packages/any/s/setuptools/setuptools-0.6c11-1.src.rpm": "pkg:pypi/[email protected]",
+  "https://files.pythonhosted.org/packages/84/d8/451842a5496844bb5c7634b231a2e4caf0d867d2e25f09b840d3b07f3d4b/multi_key_dict-2.0.win32.exe": "pkg:pypi/[email protected]",
+  "https://pypi.python.org/packages/source/d/django-contrib-comments/django-contrib-comments-1.5.tar.gz": "pkg:pypi/[email protected]",
+  "https://files.pythonhosted.org/packages/40/90/df4cb5541c4f5016bbbe04dd09135c7f5af294efa3421f9ab6332cf30dc2/zc.buildout.languageserver-0.6.2.tar.gz": "pkg:pypi/[email protected]",
+  "https://files.pythonhosted.org/packages/b0/42/cac00d0570ff45c8d3b66aa32bf1aba7a527e5908123b0164e42f6af6ae1/zc.buildout.languageserver-0.6.2-py3-none-any.whl": "pkg:pypi/[email protected]",
   "http://nuget.org/packages/EntityFramework/4.2.0.0": "pkg:nuget/[email protected]",
   "http://www.nuget.org/packages/SharpGIS.GZipWebClient/1.2.0": "pkg:nuget/[email protected]",
   "https://www.nuget.org/api/v2/package/Newtonsoft.Json/11.0.1": "pkg:nuget/[email protected]",
@@ -145,18 +151,18 @@
   "http://master.dl.sourceforge.net/project/wxmozilla/wxMozilla/0.5.5/wxMozilla-0.5.5.exe": "pkg:sourceforge/wxmozilla/[email protected]",
   "http://iweb.dl.sourceforge.net/project/sblim/sblim-cim-client2/2.2.5/sblim-cim-client2-2.2.5-src.zip": "pkg:sourceforge/sblim/[email protected]",
   "http://master.dl.sourceforge.net/project/zinnia/zinnia-win32/0.06/zinnia-win32-0.06.zip": "pkg:sourceforge/zinnia/[email protected]",
-  "http://iweb.dl.sourceforge.net/project/findbugs/findbugs/1.3.4/findbugs-1.3.4.tar.gz/": "pkg:sourceforge/findbugs-1.3.4.tar.gz?download_url=http://iweb.dl.sourceforge.net/project/findbugs/findbugs/1.3.4/findbugs-1.3.4.tar.gz/",
-  "http://master.dl.sourceforge.net/project/arestc/net/sf/arestc/arestc/0.1.4/arestc-0.1.4-javadoc.jar": "pkg:sourceforge/arestc-0.1.4-javadoc.jar?download_url=http://master.dl.sourceforge.net/project/arestc/net/sf/arestc/arestc/0.1.4/arestc-0.1.4-javadoc.jar",
-  "http://master.dl.sourceforge.net/project/intraperson/OldFiles/intraperson/0.28/intraperson-0.28.tar.gz": "pkg:sourceforge/intraperson-0.28.tar.gz?download_url=http://master.dl.sourceforge.net/project/intraperson/OldFiles/intraperson/0.28/intraperson-0.28.tar.gz",
-  "http://master.dl.sourceforge.net/project/pwiki/pwiki/0.1.2/0.1.2.zip": "pkg:sourceforge/0.1.2.zip?download_url=http://master.dl.sourceforge.net/project/pwiki/pwiki/0.1.2/0.1.2.zip",
-  "http://master.dl.sourceforge.net/project/iswraid/iswraid/0.1.4.3/2.4.28-pre3-iswraid.patch.gz": "pkg:sourceforge/2.4.28-pre3-iswraid.patch.gz?download_url=http://master.dl.sourceforge.net/project/iswraid/iswraid/0.1.4.3/2.4.28-pre3-iswraid.patch.gz",
-  "http://master.dl.sourceforge.net/project/aloyscore/aloyscore/0.1a1%20stable/0.1a1_stable_AloysCore.zip": "pkg:sourceforge/0.1a1_stable_AloysCore.zip?download_url=http://master.dl.sourceforge.net/project/aloyscore/aloyscore/0.1a1%2520stable/0.1a1_stable_AloysCore.zip",
-  "http://master.dl.sourceforge.net/project/myenterprise/OldFiles/1.0.0.2.MyEnterprise.Source.zip": "pkg:sourceforge/1.0.0.2.MyEnterprise.Source.zip?download_url=http://master.dl.sourceforge.net/project/myenterprise/OldFiles/1.0.0.2.MyEnterprise.Source.zip",
-  "http://master.dl.sourceforge.net/project/wxhaskell/wxhaskell/wxhaskell-0.9/wxhaskell-src-0.9.zip": "pkg:sourceforge/wxhaskell-src-0.9.zip?download_url=http://master.dl.sourceforge.net/project/wxhaskell/wxhaskell/wxhaskell-0.9/wxhaskell-src-0.9.zip",
-  "http://master.dl.sourceforge.net/project/a2freedom/A2/1.2/a2freedom-1.2.zip": "pkg:sourceforge/a2freedom-1.2.zip?download_url=http://master.dl.sourceforge.net/project/a2freedom/A2/1.2/a2freedom-1.2.zip",
-  "http://master.dl.sourceforge.net/project/tinyos/OldFiles/tinyos/1.1.0/tinyos-1.1.0.tar.gz": "pkg:sourceforge/tinyos-1.1.0.tar.gz?download_url=http://master.dl.sourceforge.net/project/tinyos/OldFiles/tinyos/1.1.0/tinyos-1.1.0.tar.gz",
-  "http://master.dl.sourceforge.net/project/urlchecker/lu/ng/urlchecker/urlchecker/1.7/urlchecker-1.7-javadoc.jar": "pkg:sourceforge/urlchecker-1.7-javadoc.jar?download_url=http://master.dl.sourceforge.net/project/urlchecker/lu/ng/urlchecker/urlchecker/1.7/urlchecker-1.7-javadoc.jar",
-  "http://master.dl.sourceforge.net/project/zclasspath/maven2/org/zclasspath/zclasspath/1.5/zclasspath-1.5.jar": "pkg:sourceforge/zclasspath-1.5.jar?download_url=http://master.dl.sourceforge.net/project/zclasspath/maven2/org/zclasspath/zclasspath/1.5/zclasspath-1.5.jar",
+  "http://iweb.dl.sourceforge.net/project/findbugs/findbugs/1.3.4/findbugs-1.3.4.tar.gz/": "pkg:sourceforge/findbugs?download_url=http://iweb.dl.sourceforge.net/project/findbugs/findbugs/1.3.4/findbugs-1.3.4.tar.gz/",
+  "http://master.dl.sourceforge.net/project/arestc/net/sf/arestc/arestc/0.1.4/arestc-0.1.4-javadoc.jar": "pkg:sourceforge/arestc?download_url=http://master.dl.sourceforge.net/project/arestc/net/sf/arestc/arestc/0.1.4/arestc-0.1.4-javadoc.jar",
+  "http://master.dl.sourceforge.net/project/intraperson/OldFiles/intraperson/0.28/intraperson-0.28.tar.gz": "pkg:sourceforge/intraperson?download_url=http://master.dl.sourceforge.net/project/intraperson/OldFiles/intraperson/0.28/intraperson-0.28.tar.gz",
+  "http://master.dl.sourceforge.net/project/pwiki/pwiki/0.1.2/0.1.2.zip": "pkg:sourceforge/pwiki?download_url=http://master.dl.sourceforge.net/project/pwiki/pwiki/0.1.2/0.1.2.zip",
+  "http://master.dl.sourceforge.net/project/iswraid/iswraid/0.1.4.3/2.4.28-pre3-iswraid.patch.gz": "pkg:sourceforge/iswraid?download_url=http://master.dl.sourceforge.net/project/iswraid/iswraid/0.1.4.3/2.4.28-pre3-iswraid.patch.gz",
+  "http://master.dl.sourceforge.net/project/aloyscore/aloyscore/0.1a1%20stable/0.1a1_stable_AloysCore.zip": "pkg:sourceforge/aloyscore?download_url=http://master.dl.sourceforge.net/project/aloyscore/aloyscore/0.1a1%2520stable/0.1a1_stable_AloysCore.zip",
+  "http://master.dl.sourceforge.net/project/myenterprise/OldFiles/1.0.0.2.MyEnterprise.Source.zip": "pkg:sourceforge/myenterprise?download_url=http://master.dl.sourceforge.net/project/myenterprise/OldFiles/1.0.0.2.MyEnterprise.Source.zip",
+  "http://master.dl.sourceforge.net/project/wxhaskell/wxhaskell/wxhaskell-0.9/wxhaskell-src-0.9.zip": "pkg:sourceforge/wxhaskell?download_url=http://master.dl.sourceforge.net/project/wxhaskell/wxhaskell/wxhaskell-0.9/wxhaskell-src-0.9.zip",
+  "http://master.dl.sourceforge.net/project/a2freedom/A2/1.2/a2freedom-1.2.zip": "pkg:sourceforge/a2freedom?download_url=http://master.dl.sourceforge.net/project/a2freedom/A2/1.2/a2freedom-1.2.zip",
+  "http://master.dl.sourceforge.net/project/tinyos/OldFiles/tinyos/1.1.0/tinyos-1.1.0.tar.gz": "pkg:sourceforge/tinyos?download_url=http://master.dl.sourceforge.net/project/tinyos/OldFiles/tinyos/1.1.0/tinyos-1.1.0.tar.gz",
+  "http://master.dl.sourceforge.net/project/urlchecker/lu/ng/urlchecker/urlchecker/1.7/urlchecker-1.7-javadoc.jar": "pkg:sourceforge/urlchecker?download_url=http://master.dl.sourceforge.net/project/urlchecker/lu/ng/urlchecker/urlchecker/1.7/urlchecker-1.7-javadoc.jar",
+  "http://master.dl.sourceforge.net/project/zclasspath/maven2/org/zclasspath/zclasspath/1.5/zclasspath-1.5.jar": "pkg:sourceforge/zclasspath?download_url=http://master.dl.sourceforge.net/project/zclasspath/maven2/org/zclasspath/zclasspath/1.5/zclasspath-1.5.jar",
   "https://crates.io/api/v1/crates/rand/0.7.2/download": "pkg:cargo/[email protected]",
   "https://crates.io/api/v1/crates/clap/2.33.0/download": "pkg:cargo/[email protected]",
   "https://crates.io/api/v1/crates/structopt/0.3.11/download": "pkg:cargo/[email protected]",
@@ -228,6 +234,7 @@
   "https://bitbucket.org/TG1999/first_repo/new_folder/": "pkg:bitbucket/tg1999/first_repo@new_folder",
   "https://bitbucket.org/multicoreware/x265/downloads/x265_2.6.tar.gz": "pkg:bitbucket/multicoreware/x265?download_url=https://bitbucket.org/multicoreware/x265/downloads/x265_2.6.tar.gz",
   "https://bitbucket.org/robeden/trove/downloads/trove-3.0.3.zip": "pkg:bitbucket/robeden/trove?download_url=https://bitbucket.org/robeden/trove/downloads/trove-3.0.3.zip",
+  "https://bitbucket.org/efotinis/deskpins/downloads/DeskPins-1.31-setup.exe": "pkg:bitbucket/efotinis/deskpins?download_url=https://bitbucket.org/efotinis/deskpins/downloads/DeskPins-1.31-setup.exe",
   "https://gitlab.com/TG1999/firebase/-/tree/1a122122/views": "pkg:gitlab/tg1999/firebase@1a122122#views",
   "https://gitlab.com/tg1999/firebase": "pkg:gitlab/tg1999/firebase",
   "https://gitlab.com/TG1999/firebase/-/": "pkg:gitlab/tg1999/firebase",
@@ -241,5 +248,8 @@
   "https://hackage.haskell.org/package/a50-0.5/a50-0.5.tar.gz": "pkg:hackage/[email protected]",
   "https://hackage.haskell.org/package/AC-HalfInteger-1.2.1/AC-HalfInteger-1.2.1.tar.gz": "pkg:hackage/[email protected]",
   "https://hackage.haskell.org/package/3d-graphics-examples-0.0.0.2/3d-graphics-examples-0.0.0.2.tar.gz": "pkg:hackage/[email protected]",
-  "https://salsa.debian.org/lxc-team/lxc/-/archive/master/lxc-master.tar.gz": "pkg:generic/lxc-master.tar.gz?download_url=https://salsa.debian.org/lxc-team/lxc/-/archive/master/lxc-master.tar.gz"
+  "https://salsa.debian.org/lxc-team/lxc/-/archive/master/lxc-master.tar.gz": "pkg:generic/lxc-master.tar.gz?download_url=https://salsa.debian.org/lxc-team/lxc/-/archive/master/lxc-master.tar.gz",
+  "http://apt-rpm.org/": null,
+  "": null,
+  "https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/android-notifier/android-notifier-desktop-0.5.1-1.i386.rpm": "pkg:generic/code.google.com/android-notifier?download_url=https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/android-notifier/android-notifier-desktop-0.5.1-1.i386.rpm"
 }