concord-db, concord-server: cache github app user mapping in database

agent/pom.xml

@@ -29,6 +29,10 @@
             <groupId>com.walmartlabs.concord</groupId>
             <artifactId>concord-common</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.walmartlabs.concord</groupId>
+            <artifactId>concord-github-app-installation</artifactId>
+        </dependency>
         <dependency>
             <groupId>com.walmartlabs.concord</groupId>
             <artifactId>concord-client2</artifactId>
@@ -143,6 +147,11 @@
             <artifactId>mockito-core</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-junit-jupiter</artifactId>
+            <scope>test</scope>
+        </dependency>
 
         <!-- dependency tree fix for mvnd -->
         <dependency>

agent/src/main/java/com/walmartlabs/concord/agent/AgentAuthTokenProvider.java

@@ -0,0 +1,88 @@
+package com.walmartlabs.concord.agent;
+
+/*-
+ * *****
+ * Concord
+ * -----
+ * Copyright (C) 2017 - 2025 Walmart Inc.
+ * -----
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =====
+ */
+
+import com.typesafe.config.Config;
+import com.walmartlabs.concord.agent.remote.ApiClientFactory;
+import com.walmartlabs.concord.common.AuthTokenProvider;
+import com.walmartlabs.concord.common.ExternalAuthToken;
+import com.walmartlabs.concord.github.appinstallation.GitHubAppInstallation;
+import com.walmartlabs.concord.sdk.Secret;
+
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+import java.net.URI;
+import java.util.List;
+import java.util.Optional;
+
+public class AgentAuthTokenProvider implements AuthTokenProvider {
+    private final List<AuthTokenProvider> authTokenProviders;
+
+    @Inject
+    public AgentAuthTokenProvider(GitHubAppInstallation githubProvider,
+                                  OauthTokenProvider oauthTokenProvider) {
+
+        this.authTokenProviders = List.of(
+                githubProvider,
+                oauthTokenProvider
+        );
+    }
+
+    @Override
+    public boolean supports(URI repo, @Nullable Secret secret) {
+        return authTokenProviders.stream()
+                .anyMatch(p -> p.supports(repo, secret));
+    }
+
+    public Optional<ExternalAuthToken> getToken(URI repo, @Nullable Secret secret) {
+        for (var k : authTokenProviders) {
+            if (k.supports(repo, secret)) {
+                return k.getToken(repo, secret);
+            }
+        }
+
+        return Optional.empty();
+    }
+
+    public static class ConcordServerTokenProvider implements AuthTokenProvider {
+        private final ApiClientFactory apiClientFactory;
+        private final Config config;
+
+        @Inject
+        public ConcordServerTokenProvider(ApiClientFactory apiClientFactory, Config config) {
+            this.apiClientFactory = apiClientFactory;
+            this.config = config;
+        }
+
+
+        @Override
+        public boolean supports(URI repo, @Nullable Secret secret) {
+            // TODO implement
+            return false;
+        }
+
+        @Override
+        public Optional<ExternalAuthToken> getToken(URI repo, @Nullable Secret secret) {
+            // TODO implement
+            return Optional.empty();
+        }
+    }
+}

agent/src/main/java/com/walmartlabs/concord/agent/AgentModule.java

@@ -30,7 +30,9 @@
 import com.walmartlabs.concord.agent.remote.ApiClientFactory;
 import com.walmartlabs.concord.agent.remote.QueueClientProvider;
 import com.walmartlabs.concord.common.ObjectMapperProvider;
+import com.walmartlabs.concord.common.cfg.OauthTokenConfig;
 import com.walmartlabs.concord.config.ConfigModule;
+import com.walmartlabs.concord.github.appinstallation.cfg.GitHubAppInstallationConfig;
 import com.walmartlabs.concord.server.queueclient.QueueClient;
 
 import javax.inject.Named;
@@ -59,6 +61,10 @@ public void configure(Binder binder) {
         binder.bind(DockerConfiguration.class).in(SINGLETON);
         binder.bind(RuntimeConfiguration.class).asEagerSingleton();
         binder.bind(GitConfiguration.class).in(SINGLETON);
+        binder.bind(OauthTokenConfig.class).to(GitConfiguration.class).in(SINGLETON);
+        binder.bind(GitHubConfiguration.class).in(SINGLETON);
+        binder.bind(GitHubAppInstallationConfig.class).to(GitHubConfiguration.class).in(SINGLETON);
+        binder.bind(AgentAuthTokenProvider.ConcordServerTokenProvider.class).in(SINGLETON);
         binder.bind(ImportConfiguration.class).in(SINGLETON);
         binder.bind(PreForkConfiguration.class).in(SINGLETON);
         binder.bind(RepositoryCacheConfiguration.class).in(SINGLETON);

agent/src/main/java/com/walmartlabs/concord/agent/RepositoryManager.java

@@ -51,13 +51,14 @@ public RepositoryManager(SecretClient secretClient,
                              GitConfiguration gitCfg,
                              RepositoryCacheConfiguration cacheCfg,
                              ObjectMapper objectMapper,
-                             DependencyManager dependencyManager) throws IOException {
+                             DependencyManager dependencyManager,
+                             AgentAuthTokenProvider agentAuthTokenProvider) throws IOException {
 
         this.secretClient = secretClient;
         this.gitCfg = gitCfg;
 
         GitClientConfiguration clientCfg = GitClientConfiguration.builder()
-                .oauthToken(gitCfg.getToken())
+                .oauthToken(gitCfg.getOauthToken())
                 .defaultOperationTimeout(gitCfg.getDefaultOperationTimeout())
                 .fetchTimeout(gitCfg.getFetchTimeout())
                 .httpLowSpeedLimit(gitCfg.getHttpLowSpeedLimit())
@@ -66,9 +67,10 @@ public RepositoryManager(SecretClient secretClient,
                 .sshTimeoutRetryCount(gitCfg.getSshTimeoutRetryCount())
                 .build();
 
-        List<RepositoryProvider> providers = Arrays.asList(new MavenRepositoryProvider(dependencyManager), new GitCliRepositoryProvider(clientCfg));
-        this.providers = new RepositoryProviders(providers);
-
+        this.providers = new RepositoryProviders(List.of(
+                new MavenRepositoryProvider(dependencyManager),
+                new GitCliRepositoryProvider(clientCfg, agentAuthTokenProvider)
+        ));
         this.repositoryCache = new RepositoryCache(cacheCfg.getCacheDir(),
                 cacheCfg.getInfoDir(),
                 cacheCfg.getLockTimeout(),

agent/src/main/java/com/walmartlabs/concord/agent/cfg/GitConfiguration.java

@@ -21,15 +21,21 @@
  */
 
 import com.typesafe.config.Config;
+import com.walmartlabs.concord.common.cfg.MappingAuthConfig;
+import com.walmartlabs.concord.common.cfg.OauthTokenConfig;
 
 import javax.inject.Inject;
 import java.time.Duration;
+import java.util.List;
+import java.util.Optional;
 
 import static com.walmartlabs.concord.agent.cfg.Utils.getStringOrDefault;
 
-public class GitConfiguration {
+public class GitConfiguration implements OauthTokenConfig {
 
     private final String token;
+    private final String oauthUsername;
+    private final String oauthUrlPattern;
     private final boolean shallowClone;
     private final boolean checkAlreadyFetched;
     private final Duration defaultOperationTimeout;
@@ -39,10 +45,13 @@ public class GitConfiguration {
     private final Duration sshTimeout;
     private final int sshTimeoutRetryCount;
     private final boolean skip;
+    private final List<? extends Config> authConfigs;
 
     @Inject
     public GitConfiguration(Config cfg) {
         this.token = getStringOrDefault(cfg, "git.oauth", () -> null);
+        this.oauthUsername = getStringOrDefault(cfg, "git.oauthUsername", () -> null);
+        this.oauthUrlPattern = getStringOrDefault(cfg, "git.oauthUrlPattern", () -> null);
         this.shallowClone = cfg.getBoolean("git.shallowClone");
         this.checkAlreadyFetched = cfg.getBoolean("git.checkAlreadyFetched");
         this.defaultOperationTimeout = cfg.getDuration("git.defaultOperationTimeout");
@@ -52,10 +61,22 @@ public GitConfiguration(Config cfg) {
         this.sshTimeout = cfg.getDuration("git.sshTimeout");
         this.sshTimeoutRetryCount = cfg.getInt("git.sshTimeoutRetryCount");
         this.skip = cfg.getBoolean("git.skip");
+        this.authConfigs = cfg.getConfigList("git.systemAuth");
     }
 
-    public String getToken() {
-        return token;
+    @Override
+    public Optional<String> getOauthToken() {
+        return Optional.ofNullable(token);
+    }
+
+    @Override
+    public Optional<String> getOauthUsername() {
+        return Optional.ofNullable(oauthUsername);
+    }
+
+    @Override
+    public Optional<String> getOauthUrlPattern() {
+        return Optional.ofNullable(oauthUrlPattern);
     }
 
     public boolean isShallowClone() {
@@ -93,4 +114,46 @@ public int getSshTimeoutRetryCount() {
     public boolean isSkip() {
         return skip;
     }
+
+    public List<MappingAuthConfig> getSystemAuth() {
+        return authConfigs.stream()
+                .map(o -> {
+                    AuthSource type = AuthSource.valueOf(o.getString("type").toUpperCase());
+
+                    return (AuthConfig) switch (type) {
+                        case OAUTH_TOKEN -> OauthConfig.from(o);
+                    };
+                })
+                .map(AuthConfig::toGitAuth)
+                .toList();
+
+    }
+
+    enum AuthSource {
+        OAUTH_TOKEN
+    }
+
+    public interface AuthConfig {
+        MappingAuthConfig toGitAuth();
+    }
+
+    public record OauthConfig(String id, String urlPattern, String token) implements AuthConfig {
+
+        static OauthConfig from(Config cfg) {
+            return new OauthConfig(
+                    getStringOrDefault(cfg, "id", () -> "system-oauth-token"),
+                    cfg.getString("urlPattern"),
+                    cfg.getString("token")
+            );
+        }
+
+        @Override
+        public MappingAuthConfig.OauthAuthConfig toGitAuth() {
+            return MappingAuthConfig.OauthAuthConfig.builder()
+                    .id(this.id())
+                    .urlPattern(MappingAuthConfig.assertBaseUrlPattern(this.urlPattern()))
+                    .token(this.token())
+                    .build();
+        }
+    }
 }

agent/src/main/java/com/walmartlabs/concord/agent/cfg/GitHubConfiguration.java

@@ -0,0 +1,63 @@
+package com.walmartlabs.concord.agent.cfg;
+
+/*-
+ * *****
+ * Concord
+ * -----
+ * Copyright (C) 2017 - 2025 Walmart Inc.
+ * -----
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =====
+ */
+
+import com.walmartlabs.concord.common.cfg.MappingAuthConfig;
+import com.walmartlabs.concord.github.appinstallation.cfg.GitHubAppInstallationConfig;
+
+import javax.inject.Inject;
+import java.time.Duration;
+import java.util.List;
+
+public class GitHubConfiguration implements GitHubAppInstallationConfig {
+
+    private static final String CFG_APP_INSTALLATION = "github.appInstallation";
+
+    private final GitHubAppInstallationConfig appInstallation;
+
+    @Inject
+    public GitHubConfiguration(com.typesafe.config.Config config) {
+        if (config.hasPath(CFG_APP_INSTALLATION)) {
+            var raw = config.getConfig(CFG_APP_INSTALLATION);
+            this.appInstallation = GitHubAppInstallationConfig.fromConfig(raw);
+        } else {
+            this.appInstallation = GitHubAppInstallationConfig.builder()
+                    .authConfigs(List.of())
+                    .build();
+        }
+    }
+
+    @Override
+    public List<MappingAuthConfig> getAuthConfigs() {
+        return appInstallation.getAuthConfigs();
+    }
+
+    @Override
+    public Duration getSystemAuthCacheDuration() {
+        return appInstallation.getSystemAuthCacheDuration();
+    }
+
+    @Override
+    public long getSystemAuthCacheMaxWeight() {
+        return appInstallation.getSystemAuthCacheMaxWeight();
+    }
+
+}

agent/src/main/resources/concord-agent.conf

@@ -188,9 +188,26 @@ concord-agent {
         # if true, skip Git fetch, use workspace state only
         skip = false
 
-        # GitHub auth token to use when cloning repositories without explicitly configured authentication
+        # GitHub auth token to use when cloning repositories without explicitly
+        # configured authentication. Deprecated in favor of systemAuth list of
+        # tokens or service-specific app config (e.g. github)
         # oauth = "..."
 
+        # specific username to use for auth
+        # oauthUsername = ""
+
+        # regex to match against git server's hostname + port + path so oauth
+        # token isn't used for and unexpected host
+        # oauthUrlPattern = ""
+
+        # List of system-provided auth token configs
+        # {
+        #   "token" = "...",
+        #   "username" = "...", # optional, username to send with auth token
+        #   "urlPattern" = "..." # required, regex to match against target git host + port + path
+        # }
+        systemAuth = []
+
         # use GIT's shallow clone
         shallowClone = true
 
@@ -212,6 +229,32 @@ concord-agent {
         sshTimeout = "10 minutes"
     }
 
+       # github app settings. While this works on the agent, it's preferable to
+       # get auth token from concord-server via externalTokenProvider
+       github {
+           # App installation settings. Multiple auth (private key) definitions are supported,
+           # as each is matched to a particular url pattern.
+           appInstallation {
+               # {
+               #     type = "GITHUB_APP_INSTALLATION",
+               #     urlPattern = "github.com",  # regex
+               #     username = "...",  # optional, defaults to "x-access-token"
+               #     apiUrl = "https://api.github.com", # github api url, usually *not* the same as the repo url host/path
+               #     clientId = "...",
+               #     privateKey = "/path/to/pk.pem"
+               # }
+               # or static oauth config. Not exactly a "GitHub App", but can do some
+               # API interactions and cloning. Less preferred to actual app.
+               # {
+               #     type = "OAUTH_TOKEN",
+               #     token = "...",
+               #     username = "...",  # optional, usually not necessary
+               #     urlPattern = "..." # regex to match against git server's hostname + port + path
+               # }
+               auth = []
+           }
+       }
+
     imports {
         # base git url for imports
         src = ""