Download the latest version in //third_party/android_deps

This cl causes the 3pp bot to download the latest version of a dep
rather than a specific one specified in build.gradle. It does not
actually roll said latest version but this should allow you to roll to
the latest version of a package in one commit (since the 3pp bot would
have already downloaded it).

Bug: 1223420
Change-Id: Ibd5eef567b80a1b3abcb19aeaab30f00a0b67c56
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3421236
Reviewed-by: Andrew Grieve <[email protected]>
Reviewed-by: Haiyang Pan <[email protected]>
Auto-Submit: Mohamed Heikal <[email protected]>
Commit-Queue: Mohamed Heikal <[email protected]>
Cr-Commit-Position: refs/heads/main@{#965274}

third_party/android_deps/README.md

@@ -28,30 +28,47 @@ Full steps to add a new third party library or update existing libraries:
 1. Update `build.gradle` with the new dependency or the new versions.
 
 2. Run `fetch_all.py` to update your current workspace with the changes. This
-   will update, among other things, your top-level DEPS file.
+   will update, among other things, your top-level DEPS file. If this is a new
+   library, you can skip directly to step 5 since the next step is not going to
+   work for you.
 
-3. `git add` all the 3pp related changes and create a CL for review. Keep the
+3. Run `gclient sync` to make sure that cipd has access to the versions you are
+   trying to roll. This might fail with a cipd error failing to resolve a tag.
+
+4. If the previous step works, upload your cl and you are done, if not continue
+   with the steps.
+
+5. Add a `overrideLatest` property override to your package in
+   `ChromiumDepGraph.groovy` in the `PROPERTY_OVERRIDES` map, set it to `true`.
+
+6. Run `fetch_all.py` again.
+
+7. `git add` all the 3pp related changes and create a CL for review. Keep the
    `3pp/`, `.gradle`, `OWNERS`, `.groovy` changes in the CL and revert the other
    files. The other files should be committed in a follow up CL. Example git commands:
    * `git add third_party/android_deps{*.gradle,*.groovy,*3pp*,*OWNERS,*README.md}`
    * `git commit -m commit_message`
    * `git restore third_party/android_deps DEPS`
    * `git clean -id`
 
-4. Land the first CL in step 3 and wait for the corresponding 3pp packager to
-   create the new CIPD packages. The 3pp packager runs every 6 hours. You can
-   see the latest runs [here][3pp_bot]. See
+8. Land the first CL in the previous step and wait for the corresponding 3pp
+   packager to create the new CIPD packages. The 3pp packager runs every 6
+   hours.  You can see the latest runs [here][3pp_bot]. See
    [`//docs/cipd_and_3pp.md`][cipd_and_3pp_doc] for how it works. Anyone on the
    Clank build core team and any trooper can trigger the bot on demand for you.
 
-5. If your follow up CL takes more than a day please revert the original CL.
+9. If your follow up CL takes more than a day please revert the original CL.
    Once the bot uploads to cipd there is no need to keep the modified 3pp files.
    The bot runs 4 times a day. When you are ready to land the follow up CL, you
    can land everything together since the cipd packages have already been
    uploaded.
 
-6. Run `fetch_all.py` again. There should not be any 3pp related changes. Create
-   a commit.
+10. Remove your `overrideLatest` property override entry in
+    `ChromiumDepGraph.groovy` so that the 3pp bot goes back to downloading and
+    storing the latest versions of your package so that it is available when you
+    next try to roll.
+
+11. Run `fetch_all.py` again. Create a CL with the changes and land it.
 
    If the CL is doing more than upgrading existing packages or adding packages
    from the same source and license (e.g. gms) follow

third_party/android_deps/buildSrc/src/main/groovy/3ppFetch.template

@@ -8,28 +8,44 @@ from __future__ import print_function
 import argparse
 import json
 import os
+import re
+import urllib2
 
-_FILE_URL = '${dependency.fileUrl}'
-_FILE_NAME = '${dependency.fileName}'
-_FILE_VERSION = '${dependency.version}'
+_REPO_URL = '${dependency.repoUrl}'
+_GROUP_NAME = '${dependency.group.replace('.', '/')}'
+_MODULE_NAME = '${dependency.name}'
+_FILE_EXT = '${dependency.extension}'
+_OVERRIDE_LATEST = <%= dependency.overrideLatest ? "'${dependency.version}'" : "None" %>
 
 
 def do_latest():
-    print(_FILE_VERSION)
+    if _OVERRIDE_LATEST is not None:
+        print(_OVERRIDE_LATEST)
+        return
+    maven_metadata_url = '{}/{}/{}/maven-metadata.xml'.format(
+        _REPO_URL, _GROUP_NAME, _MODULE_NAME)
+    metadata = urllib2.urlopen(maven_metadata_url).read()
+    # Do not parse xml with the python included parser since it is susceptible
+    # to maliciously crafted xmls. Only use regular expression parsing to be
+    # safe. RE should be enough to handle what we need to extract.
+    match = re.search('<latest>([^<]+)</latest>', metadata)
+    if match:
+        latest = match.group(1)
+    else:
+        # if no latest info was found just hope the versions are sorted and the
+        # last one is the latest (as is commonly the case).
+        latest = re.findall('<version>([^<]+)</version>', metadata)[-1]
+    print(latest)
 
 
 def get_download_url(version):
-    if _FILE_URL.endswith('.jar'):
-        ext = '.jar'
-    elif _FILE_URL.endswith('.aar'):
-        ext = '.aar'
-    else:
-        raise Exception('Unsupported extension for %s' % _FILE_URL)
+    file_url = '{0}/{1}/{2}/{3}/{2}-{3}.{4}'.format(_REPO_URL, _GROUP_NAME,
+                                                    _MODULE_NAME, version,
+                                                    _FILE_EXT)
 
     partial_manifest = {
-        'url': [_FILE_URL],
-        'name': [_FILE_NAME],
-        'ext': ext,
+        'url': [file_url],
+        'ext': '.' + _FILE_EXT,
     }
     print(json.dumps(partial_manifest))
 

third_party/android_deps/buildSrc/src/main/groovy/ChromiumDepGraph.groovy

@@ -127,7 +127,8 @@ class ChromiumDepGraph {
             description: 'Only contains necessary framework & Xerces2 classes',
             url: 'http://nekohtml.sourceforge.net/index.html',
             licenseUrl: 'https://www.apache.org/licenses/LICENSE-2.0.txt',
-            licenseName: 'Apache 2.0'),
+            licenseName: 'Apache 2.0',
+            overrideLatest: true),
         org_apache_ant_ant: new PropertyOverride(
             url: 'https://ant.apache.org/',
             licenseUrl: 'https://www.apache.org/licenses/LICENSE-2.0.txt',
@@ -521,9 +522,9 @@ class ChromiumDepGraph {
 
     private DependencyDescription buildDepDescription(
             String id, ResolvedDependency dependency, ResolvedArtifact artifact, List<String> childModules) {
-        String pomUrl
+        String pomUrl, repoUrl
         GPathResult pomContent
-        (pomUrl, pomContent) = computePomFromArtifact(artifact)
+        (repoUrl, pomUrl, pomContent) = computePomFromArtifact(artifact)
 
         List<LicenseSpec> licenses = []
         if (!skipLicenses) {
@@ -552,6 +553,7 @@ class ChromiumDepGraph {
                 directoryName: id.toLowerCase(),
                 fileName: artifact.file.name,
                 fileUrl: fileUrl,
+                repoUrl: repoUrl,
                 description: description,
                 url: pomContent.url?.text(),
                 displayName: displayName,
@@ -615,6 +617,7 @@ class ChromiumDepGraph {
                 description = fallbackProperties.description ?: description
                 url = fallbackProperties.url ?: url
                 cipdSuffix = fallbackProperties.cipdSuffix ?: cipdSuffix
+                overrideLatest = fallbackProperties.overrideLatest ?: false
                 // Boolean properties require explicit null checks instead of only when truish.
                 if (fallbackProperties.generateTarget != null) {
                     generateTarget = fallbackProperties.generateTarget
@@ -701,7 +704,7 @@ class ChromiumDepGraph {
                 GPathResult content = new XmlSlurper(
                         false /* validating */, false /* namespaceAware */).parse(fileUrl)
                 logger.debug("Succeeded in resolving url $fileUrl")
-                return [fileUrl, content]
+                return [repoUrl, fileUrl, content]
             } catch (any) {
                 logger.debug("Failed in resolving url $fileUrl")
             }
@@ -730,6 +733,10 @@ class ChromiumDepGraph {
         String group, name, version, extension, displayName, description, url
         List<LicenseSpec> licenses
         String fileName, fileUrl
+        // |repoUrl| is the url to the repo that hosts this dep's artifact
+        // (|fileUrl|). Basically |fileurl|.startswith(|repoUrl|). |url| is the
+        // project homepage as supplied by the developer.
+        String repoUrl
         // The local directory name to store the files like artifact, license file, 3pp subdirectory, and etc. Must be
         // lowercase since 3pp uses the directory name as part of the CIPD names. However CIPD does not allow uppercase
         // in names.
@@ -740,6 +747,10 @@ class ChromiumDepGraph {
         ComponentIdentifier componentId
         List<String> children
         String cipdSuffix
+        // When set overrides the version downloaded by the 3pp fetch script to
+        // be, instead of the latest available, the resolved version by gradle
+        // in this run.
+        Boolean overrideLatest
 
     }
 
@@ -760,6 +771,9 @@ class ChromiumDepGraph {
         Boolean exclude
         // Set to false to skip creation of BUILD.gn target.
         Boolean generateTarget
+        // Set to override the 3pp fetch script returing the latest version and
+        // instead forcibly return the version required by gradle.
+        Boolean overrideLatest
 
     }
 

third_party/android_deps/libs/android_arch_core_common/3pp/fetch.py

@@ -11,28 +11,44 @@
 import argparse
 import json
 import os
+import re
+import urllib2
 
-_FILE_URL = 'https://dl.google.com/dl/android/maven2/android/arch/core/common/1.1.1/common-1.1.1.jar'
-_FILE_NAME = 'common-1.1.1.jar'
-_FILE_VERSION = '1.1.1'
+_REPO_URL = 'https://dl.google.com/dl/android/maven2'
+_GROUP_NAME = 'android/arch/core'
+_MODULE_NAME = 'common'
+_FILE_EXT = 'jar'
+_OVERRIDE_LATEST = None
 
 
 def do_latest():
-    print(_FILE_VERSION)
+    if _OVERRIDE_LATEST is not None:
+        print(_OVERRIDE_LATEST)
+        return
+    maven_metadata_url = '{}/{}/{}/maven-metadata.xml'.format(
+        _REPO_URL, _GROUP_NAME, _MODULE_NAME)
+    metadata = urllib2.urlopen(maven_metadata_url).read()
+    # Do not parse xml with the python included parser since it is susceptible
+    # to maliciously crafted xmls. Only use regular expression parsing to be
+    # safe. RE should be enough to handle what we need to extract.
+    match = re.search('<latest>([^<]+)</latest>', metadata)
+    if match:
+        latest = match.group(1)
+    else:
+        # if no latest info was found just hope the versions are sorted and the
+        # last one is the latest (as is commonly the case).
+        latest = re.findall('<version>([^<]+)</version>', metadata)[-1]
+    print(latest)
 
 
 def get_download_url(version):
-    if _FILE_URL.endswith('.jar'):
-        ext = '.jar'
-    elif _FILE_URL.endswith('.aar'):
-        ext = '.aar'
-    else:
-        raise Exception('Unsupported extension for %s' % _FILE_URL)
+    file_url = '{0}/{1}/{2}/{3}/{2}-{3}.{4}'.format(_REPO_URL, _GROUP_NAME,
+                                                    _MODULE_NAME, version,
+                                                    _FILE_EXT)
 
     partial_manifest = {
-        'url': [_FILE_URL],
-        'name': [_FILE_NAME],
-        'ext': ext,
+        'url': [file_url],
+        'ext': '.' + _FILE_EXT,
     }
     print(json.dumps(partial_manifest))
 

third_party/android_deps/libs/android_arch_core_runtime/3pp/fetch.py

@@ -11,28 +11,44 @@
 import argparse
 import json
 import os
+import re
+import urllib2
 
-_FILE_URL = 'https://dl.google.com/dl/android/maven2/android/arch/core/runtime/1.1.1/runtime-1.1.1.aar'
-_FILE_NAME = 'runtime-1.1.1.aar'
-_FILE_VERSION = '1.1.1'
+_REPO_URL = 'https://dl.google.com/dl/android/maven2'
+_GROUP_NAME = 'android/arch/core'
+_MODULE_NAME = 'runtime'
+_FILE_EXT = 'aar'
+_OVERRIDE_LATEST = None
 
 
 def do_latest():
-    print(_FILE_VERSION)
+    if _OVERRIDE_LATEST is not None:
+        print(_OVERRIDE_LATEST)
+        return
+    maven_metadata_url = '{}/{}/{}/maven-metadata.xml'.format(
+        _REPO_URL, _GROUP_NAME, _MODULE_NAME)
+    metadata = urllib2.urlopen(maven_metadata_url).read()
+    # Do not parse xml with the python included parser since it is susceptible
+    # to maliciously crafted xmls. Only use regular expression parsing to be
+    # safe. RE should be enough to handle what we need to extract.
+    match = re.search('<latest>([^<]+)</latest>', metadata)
+    if match:
+        latest = match.group(1)
+    else:
+        # if no latest info was found just hope the versions are sorted and the
+        # last one is the latest (as is commonly the case).
+        latest = re.findall('<version>([^<]+)</version>', metadata)[-1]
+    print(latest)
 
 
 def get_download_url(version):
-    if _FILE_URL.endswith('.jar'):
-        ext = '.jar'
-    elif _FILE_URL.endswith('.aar'):
-        ext = '.aar'
-    else:
-        raise Exception('Unsupported extension for %s' % _FILE_URL)
+    file_url = '{0}/{1}/{2}/{3}/{2}-{3}.{4}'.format(_REPO_URL, _GROUP_NAME,
+                                                    _MODULE_NAME, version,
+                                                    _FILE_EXT)
 
     partial_manifest = {
-        'url': [_FILE_URL],
-        'name': [_FILE_NAME],
-        'ext': ext,
+        'url': [file_url],
+        'ext': '.' + _FILE_EXT,
     }
     print(json.dumps(partial_manifest))
 

third_party/android_deps/libs/android_arch_lifecycle_common/3pp/fetch.py

@@ -11,28 +11,44 @@
 import argparse
 import json
 import os
+import re
+import urllib2
 
-_FILE_URL = 'https://dl.google.com/dl/android/maven2/android/arch/lifecycle/common/1.1.1/common-1.1.1.jar'
-_FILE_NAME = 'common-1.1.1.jar'
-_FILE_VERSION = '1.1.1'
+_REPO_URL = 'https://dl.google.com/dl/android/maven2'
+_GROUP_NAME = 'android/arch/lifecycle'
+_MODULE_NAME = 'common'
+_FILE_EXT = 'jar'
+_OVERRIDE_LATEST = None
 
 
 def do_latest():
-    print(_FILE_VERSION)
+    if _OVERRIDE_LATEST is not None:
+        print(_OVERRIDE_LATEST)
+        return
+    maven_metadata_url = '{}/{}/{}/maven-metadata.xml'.format(
+        _REPO_URL, _GROUP_NAME, _MODULE_NAME)
+    metadata = urllib2.urlopen(maven_metadata_url).read()
+    # Do not parse xml with the python included parser since it is susceptible
+    # to maliciously crafted xmls. Only use regular expression parsing to be
+    # safe. RE should be enough to handle what we need to extract.
+    match = re.search('<latest>([^<]+)</latest>', metadata)
+    if match:
+        latest = match.group(1)
+    else:
+        # if no latest info was found just hope the versions are sorted and the
+        # last one is the latest (as is commonly the case).
+        latest = re.findall('<version>([^<]+)</version>', metadata)[-1]
+    print(latest)
 
 
 def get_download_url(version):
-    if _FILE_URL.endswith('.jar'):
-        ext = '.jar'
-    elif _FILE_URL.endswith('.aar'):
-        ext = '.aar'
-    else:
-        raise Exception('Unsupported extension for %s' % _FILE_URL)
+    file_url = '{0}/{1}/{2}/{3}/{2}-{3}.{4}'.format(_REPO_URL, _GROUP_NAME,
+                                                    _MODULE_NAME, version,
+                                                    _FILE_EXT)
 
     partial_manifest = {
-        'url': [_FILE_URL],
-        'name': [_FILE_NAME],
-        'ext': ext,
+        'url': [file_url],
+        'ext': '.' + _FILE_EXT,
     }
     print(json.dumps(partial_manifest))