From 8e154c6f949ae5310dce39f7475c61a57eaa2d44 Mon Sep 17 00:00:00 2001 From: Michael Schuster Date: Wed, 22 Jan 2025 10:46:09 +0100 Subject: [PATCH 1/9] Allow updating code repository using the CLI --- src/zenml/cli/code_repository.py | 66 ++++++++++++++++++++++++++++++- src/zenml/client.py | 68 +++++++++++++++++++++++--------- 2 files changed, 115 insertions(+), 19 deletions(-) diff --git a/src/zenml/cli/code_repository.py b/src/zenml/cli/code_repository.py index d0b90dde4bf..b622884f153 100644 --- a/src/zenml/cli/code_repository.py +++ b/src/zenml/cli/code_repository.py @@ -42,7 +42,7 @@ def code_repository() -> None: context_settings={"ignore_unknown_options": True}, help="Register a code repository.", ) -@click.argument("name", type=click.STRING) +@click.argument("name", type=str) @click.option( "--type", "-t", @@ -183,6 +183,70 @@ def list_code_repositories(**kwargs: Any) -> None: ) +@code_repository.command("update", help="Update a code repository.") +@click.argument("name_or_id", type=str, required=True) +@click.option( + "--name", + "-n", + type=str, + required=False, + help="The new code repository name.", +) +@click.option( + "--description", + "-d", + type=str, + required=False, + help="The new code repository description.", +) +@click.option( + "--logo-url", + "-l", + type=str, + required=False, + help="New URL of a logo (png, jpg or svg) for the code repository.", +) +@click.argument( + "args", + nargs=-1, + type=click.UNPROCESSED, +) +def update_code_repository( + name_or_id: str, + name: Optional[str], + description: Optional[str], + logo_url: Optional[str], + args: List[str], +) -> None: + """Update a code repository. + + Args: + name_or_id: Name or ID of the code repository to update. + name: New name of the code repository. + description: New description of the code repository. + logo_url: New logo URL of the code repository. + args: Code repository configurations. + """ + parsed_name_or_id, parsed_args = cli_utils.parse_name_and_extra_arguments( + list(args) + [name_or_id], expand_args=True, name_mandatory=True + ) + assert parsed_name_or_id + + with console.status( + f"Updating code repository '{parsed_name_or_id}'...\n" + ): + Client().update_code_repository( + name_id_or_prefix=parsed_name_or_id, + name=name, + description=description, + logo_url=logo_url, + config=parsed_args, + ) + cli_utils.declare( + f"Successfully updated code repository `{parsed_name_or_id}`." + ) + + @code_repository.command("delete") @click.argument("name_or_id", type=str, required=True) @click.option( diff --git a/src/zenml/client.py b/src/zenml/client.py index 98ef7047df0..e9cdc29e6fa 100644 --- a/src/zenml/client.py +++ b/src/zenml/client.py @@ -4952,25 +4952,15 @@ def restore_secrets( # --------------------------- Code repositories --------------------------- - def create_code_repository( - self, - name: str, - config: Dict[str, Any], - source: Source, - description: Optional[str] = None, - logo_url: Optional[str] = None, - ) -> CodeRepositoryResponse: - """Create a new code repository. + @staticmethod + def _validate_code_repository_config( + source: Source, config: Dict[str, Any] + ) -> None: + """Validate a code repository config. Args: - name: Name of the code repository. - config: The configuration for the code repository. - source: The code repository implementation source. - description: The code repository description. - logo_url: URL of a logo (png, jpg or svg) for the code repository. - - Returns: - The created code repository. + source: The code repository source. + config: The code repository config. Raises: RuntimeError: If the provided config is invalid. @@ -4983,13 +4973,38 @@ def create_code_repository( ) ) try: - # Validate the repo config + # This does a login to verify the credentials code_repo_class(id=uuid4(), config=config) + + # Explicitly access the config for pydantic validation, in case + # the login for some reason did not do that. + _ = code_repo_class.config except Exception as e: raise RuntimeError( "Failed to validate code repository config." ) from e + def create_code_repository( + self, + name: str, + config: Dict[str, Any], + source: Source, + description: Optional[str] = None, + logo_url: Optional[str] = None, + ) -> CodeRepositoryResponse: + """Create a new code repository. + + Args: + name: Name of the code repository. + config: The configuration for the code repository. + source: The code repository implementation source. + description: The code repository description. + logo_url: URL of a logo (png, jpg or svg) for the code repository. + + Returns: + The created code repository. + """ + self._validate_code_repository_config(source=source, config=config) repo_request = CodeRepositoryRequest( user=self.active_user.id, workspace=self.active_workspace.id, @@ -5088,6 +5103,7 @@ def update_code_repository( name: Optional[str] = None, description: Optional[str] = None, logo_url: Optional[str] = None, + config: Optional[Dict[str, Any]] = None, ) -> CodeRepositoryResponse: """Update a code repository. @@ -5097,6 +5113,10 @@ def update_code_repository( name: New name of the code repository. description: New description of the code repository. logo_url: New logo URL of the code repository. + config: New configuration options for the code repository. Will + be used to update the existing configuration values. To remove + values from the existing configuration, set the value for that + key to `None`. Returns: The updated code repository. @@ -5107,6 +5127,18 @@ def update_code_repository( update = CodeRepositoryUpdate( name=name, description=description, logo_url=logo_url ) + if config is not None: + combined_config = repo.config + combined_config.update(config) + combined_config = { + k: v for k, v in combined_config.items() if v is not None + } + + self._validate_code_repository_config( + source=repo.source, config=combined_config + ) + update.config = combined_config + return self.zen_store.update_code_repository( code_repository_id=repo.id, update=update ) From e94deb5676d191443854367f21823773634ee3fb Mon Sep 17 00:00:00 2001 From: Michael Schuster Date: Wed, 22 Jan 2025 11:13:56 +0100 Subject: [PATCH 2/9] Increase logging verbosity on non-import related code repo exceptions --- src/zenml/utils/code_repository_utils.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/zenml/utils/code_repository_utils.py b/src/zenml/utils/code_repository_utils.py index 8624fe1c756..756bd9873d9 100644 --- a/src/zenml/utils/code_repository_utils.py +++ b/src/zenml/utils/code_repository_utils.py @@ -109,11 +109,16 @@ def find_active_code_repository( for model in depaginate(list_method=Client().list_code_repositories): try: repo = BaseCodeRepository.from_model(model) - except Exception: + except ImportError: logger.debug( - "Failed to instantiate code repository class.", exc_info=True + "Failed to import code repository class.", exc_info=True ) continue + except Exception: + logger.warning( + "Failed to instantiate or login to code repository.", + exc_info=True, + ) local_context = repo.get_local_context(path) if local_context: From 302b8bd3e2d29469bce6e8097c8f20a956fa5430 Mon Sep 17 00:00:00 2001 From: Michael Schuster Date: Wed, 22 Jan 2025 11:49:39 +0100 Subject: [PATCH 3/9] Some fixes and logs --- src/zenml/cli/code_repository.py | 6 ++- .../gitlab_code_repository.py | 2 +- src/zenml/pipelines/build_utils.py | 44 +++++++++++++++++-- src/zenml/utils/code_repository_utils.py | 8 ++-- .../utils/pipeline_docker_image_builder.py | 3 -- .../schemas/code_repository_schemas.py | 3 ++ .../test_pipeline_docker_image_builder.py | 2 - 7 files changed, 55 insertions(+), 13 deletions(-) diff --git a/src/zenml/cli/code_repository.py b/src/zenml/cli/code_repository.py index b622884f153..1646ebc3c5c 100644 --- a/src/zenml/cli/code_repository.py +++ b/src/zenml/cli/code_repository.py @@ -183,7 +183,11 @@ def list_code_repositories(**kwargs: Any) -> None: ) -@code_repository.command("update", help="Update a code repository.") +@code_repository.command( + "update", + help="Update a code repository.", + context_settings={"ignore_unknown_options": True}, +) @click.argument("name_or_id", type=str, required=True) @click.option( "--name", diff --git a/src/zenml/integrations/gitlab/code_repositories/gitlab_code_repository.py b/src/zenml/integrations/gitlab/code_repositories/gitlab_code_repository.py index c3e889a9865..8770bea6e70 100644 --- a/src/zenml/integrations/gitlab/code_repositories/gitlab_code_repository.py +++ b/src/zenml/integrations/gitlab/code_repositories/gitlab_code_repository.py @@ -85,7 +85,7 @@ def login(self) -> None: """ try: self._gitlab_session = Gitlab( - self.config.url, private_token=self.config.token + f"https://{self.config.host}", private_token=self.config.token ) self._gitlab_session.auth() user = self._gitlab_session.user or None diff --git a/src/zenml/pipelines/build_utils.py b/src/zenml/pipelines/build_utils.py index 810f8d5f177..a81f60892ec 100644 --- a/src/zenml/pipelines/build_utils.py +++ b/src/zenml/pipelines/build_utils.py @@ -362,6 +362,7 @@ def create_pipeline_build( item_key = checksums[checksum] image_name_or_digest = images[item_key].image contains_code = images[item_key].contains_code + requires_code_download = images[item_key].requires_code_download dockerfile = images[item_key].dockerfile requirements = images[item_key].requirements else: @@ -373,7 +374,7 @@ def create_pipeline_build( include_files = build_config.should_include_files( code_repository=code_repository, ) - download_files = build_config.should_download_files( + requires_code_download = build_config.should_download_files( code_repository=code_repository, ) pass_code_repo = ( @@ -391,7 +392,6 @@ def create_pipeline_build( tag=tag, stack=stack, include_files=include_files, - download_files=download_files, entrypoint=build_config.entrypoint, extra_files=build_config.extra_files, code_repository=code_repository if pass_code_repo else None, @@ -404,7 +404,7 @@ def create_pipeline_build( requirements=requirements, settings_checksum=checksum, contains_code=contains_code, - requires_code_download=download_files, + requires_code_download=requires_code_download, ) checksums[checksum] = combined_key @@ -537,6 +537,14 @@ def verify_local_repository_context( ) code_repository = BaseCodeRepository.from_model(model) + if will_download_from_code_repository( + deployment=deployment, local_repo_context=local_repo_context + ): + logger.info( + "Using code repository `%s` to download code for this run.", + model.name, + ) + return code_repository @@ -728,3 +736,33 @@ def should_upload_code( return True return False + + +def will_download_from_code_repository( + deployment: PipelineDeploymentBase, + local_repo_context: "LocalRepositoryContext", +) -> bool: + """Checks whether a code repository will be used to download code. + + Args: + deployment: The deployment. + local_repo_context: The local repository context. + + Returns: + Whether a code repository will be used to download code. + """ + if not build_required(deployment=deployment): + # TODO: This is not perfect as it doesn't cover wheel-based + # orchestrators + return False + + if local_repo_context.has_local_changes: + return False + + for step in deployment.step_configurations.values(): + docker_settings = step.config.docker_settings + + if docker_settings.allow_download_from_code_repository: + return True + + return False diff --git a/src/zenml/utils/code_repository_utils.py b/src/zenml/utils/code_repository_utils.py index 756bd9873d9..6371ebb7064 100644 --- a/src/zenml/utils/code_repository_utils.py +++ b/src/zenml/utils/code_repository_utils.py @@ -114,11 +114,13 @@ def find_active_code_repository( "Failed to import code repository class.", exc_info=True ) continue - except Exception: + except Exception as e: logger.warning( - "Failed to instantiate or login to code repository.", - exc_info=True, + "Failed to instantiate or login to code repository `%s`: %s", + model.name, + e, ) + continue local_context = repo.get_local_context(path) if local_context: diff --git a/src/zenml/utils/pipeline_docker_image_builder.py b/src/zenml/utils/pipeline_docker_image_builder.py index 47247c704e1..f90fa2403af 100644 --- a/src/zenml/utils/pipeline_docker_image_builder.py +++ b/src/zenml/utils/pipeline_docker_image_builder.py @@ -78,7 +78,6 @@ def build_docker_image( tag: str, stack: "Stack", include_files: bool, - download_files: bool, entrypoint: Optional[str] = None, extra_files: Optional[Dict[str, str]] = None, code_repository: Optional["BaseCodeRepository"] = None, @@ -93,7 +92,6 @@ def build_docker_image( tag: The tag to use for the image. stack: The stack on which the pipeline will be deployed. include_files: Whether to include files in the build context. - download_files: Whether to download files in the build context. entrypoint: Entrypoint to use for the final image. If left empty, no entrypoint will be included in the image. extra_files: Extra files to add to the build context. Keys are the @@ -165,7 +163,6 @@ def build_docker_image( docker_settings.apt_packages, docker_settings.environment, include_files, - download_files, entrypoint, extra_files, ] diff --git a/src/zenml/zen_stores/schemas/code_repository_schemas.py b/src/zenml/zen_stores/schemas/code_repository_schemas.py index 6bd4511edf2..d77362a19c4 100644 --- a/src/zenml/zen_stores/schemas/code_repository_schemas.py +++ b/src/zenml/zen_stores/schemas/code_repository_schemas.py @@ -151,6 +151,9 @@ def update(self, update: "CodeRepositoryUpdate") -> "CodeRepositorySchema": if update.logo_url: self.logo_url = update.logo_url + if update.config: + self.config = json.dumps(update.config) + self.updated = datetime.utcnow() return self diff --git a/tests/unit/utils/test_pipeline_docker_image_builder.py b/tests/unit/utils/test_pipeline_docker_image_builder.py index 515e889db0d..e4befd5952f 100644 --- a/tests/unit/utils/test_pipeline_docker_image_builder.py +++ b/tests/unit/utils/test_pipeline_docker_image_builder.py @@ -126,7 +126,6 @@ def test_build_skipping(): tag="tag", stack=Client().active_stack, include_files=True, - download_files=False, ) assert image_digest @@ -169,5 +168,4 @@ def test_dockerfile_needs_to_exist(): tag="tag", stack=Client().active_stack, include_files=True, - download_files=False, ) From de59cb723af60563f34cd7fe80e627e0a29d1e16 Mon Sep 17 00:00:00 2001 From: Michael Schuster Date: Wed, 22 Jan 2025 13:28:23 +0100 Subject: [PATCH 4/9] Misc improvements --- src/zenml/entrypoints/base_entrypoint_configuration.py | 8 +++++--- src/zenml/pipelines/build_utils.py | 10 ++++------ src/zenml/pipelines/pipeline_definition.py | 3 ++- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/zenml/entrypoints/base_entrypoint_configuration.py b/src/zenml/entrypoints/base_entrypoint_configuration.py index 85101ed4614..cfe9d866c38 100644 --- a/src/zenml/entrypoints/base_entrypoint_configuration.py +++ b/src/zenml/entrypoints/base_entrypoint_configuration.py @@ -214,12 +214,14 @@ def download_code_if_necessary( if not should_download_code: return - if code_reference := deployment.code_reference: + if code_path := deployment.code_path: + code_utils.download_code_from_artifact_store(code_path=code_path) + elif code_reference := deployment.code_reference: + # TODO: This might fail if the code repository had unpushed changes + # at the time the pipeline run was started. self.download_code_from_code_repository( code_reference=code_reference ) - elif code_path := deployment.code_path: - code_utils.download_code_from_artifact_store(code_path=code_path) else: raise RuntimeError( "Code download required but no code reference or path provided." diff --git a/src/zenml/pipelines/build_utils.py b/src/zenml/pipelines/build_utils.py index a81f60892ec..c1938043d8d 100644 --- a/src/zenml/pipelines/build_utils.py +++ b/src/zenml/pipelines/build_utils.py @@ -30,7 +30,6 @@ from zenml.logger import get_logger from zenml.models import ( BuildItem, - CodeReferenceRequest, PipelineBuildBase, PipelineBuildRequest, PipelineBuildResponse, @@ -703,14 +702,15 @@ def compute_stack_checksum(stack: StackResponse) -> str: def should_upload_code( deployment: PipelineDeploymentBase, build: Optional[PipelineBuildResponse], - code_reference: Optional[CodeReferenceRequest], + can_download_from_code_repository: bool, ) -> bool: """Checks whether the current code should be uploaded for the deployment. Args: deployment: The deployment. build: The build for the deployment. - code_reference: The code reference for the deployment. + can_download_from_code_repository: Whether the code can be downloaded + from a code repository. Returns: Whether the current code should be uploaded for the deployment. @@ -726,7 +726,7 @@ def should_upload_code( docker_settings = step.config.docker_settings if ( - code_reference + can_download_from_code_repository and docker_settings.allow_download_from_code_repository ): # No upload needed for this step @@ -752,8 +752,6 @@ def will_download_from_code_repository( Whether a code repository will be used to download code. """ if not build_required(deployment=deployment): - # TODO: This is not perfect as it doesn't cover wheel-based - # orchestrators return False if local_repo_context.has_local_changes: diff --git a/src/zenml/pipelines/pipeline_definition.py b/src/zenml/pipelines/pipeline_definition.py index 6580e599690..ab658f1237c 100644 --- a/src/zenml/pipelines/pipeline_definition.py +++ b/src/zenml/pipelines/pipeline_definition.py @@ -701,6 +701,7 @@ def _create_deployment( code_repository = build_utils.verify_local_repository_context( deployment=deployment, local_repo_context=local_repo_context ) + can_download_from_code_repository = code_repository is not None if prevent_build_reuse: logger.warning( @@ -737,7 +738,7 @@ def _create_deployment( if build_utils.should_upload_code( deployment=deployment, build=build_model, - code_reference=code_reference, + can_download_from_code_repository=can_download_from_code_repository, ): code_archive = code_utils.CodeArchive( root=source_utils.get_source_root() From d76ee26464570147220ee6131e48bd800bcc1e22 Mon Sep 17 00:00:00 2001 From: Michael Schuster Date: Wed, 22 Jan 2025 17:42:03 +0100 Subject: [PATCH 5/9] Docs improvements and rename some attributes for clarity --- .../connect-your-git-repository.md | 13 +++++++++---- .../code_repositories/github_code_repository.py | 13 ++++++++++--- .../code_repositories/gitlab_code_repository.py | 11 ++++++++--- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/docs/book/how-to/project-setup-and-management/setting-up-a-project-repository/connect-your-git-repository.md b/docs/book/how-to/project-setup-and-management/setting-up-a-project-repository/connect-your-git-repository.md index d2e82e82a58..28aada98648 100644 --- a/docs/book/how-to/project-setup-and-management/setting-up-a-project-repository/connect-your-git-repository.md +++ b/docs/book/how-to/project-setup-and-management/setting-up-a-project-repository/connect-your-git-repository.md @@ -48,11 +48,13 @@ Afterward, you can register a GitHub code repository by running the following CL ```shell zenml code-repository register --type=github \ ---url= --owner= --repository= \ +--owner= --repository= \ --token= ``` -where \ is the name of the code repository you are registering, \ is the owner of the repository, \ is the name of the repository, \ is your GitHub Personal Access Token and \ is the URL of the GitHub instance which defaults to `https://github.com.` You will need to set a URL if you are using GitHub Enterprise. +where \ is the name of the code repository you are registering, \ is the owner of the repository, \ is the name of the repository, \ is your GitHub Personal Access Token. + +If you're using a self-hosted GitHub Enterprise instance, you'll need to also pass the `--api_url=` and `--host=` options. \ should point to where the GitHub API is reachable (defaults to `https://api.github.com/`) and \ should be the [hostname of your GitHub instance](https://docs.github.com/en/enterprise-server@3.10/admin/configuring-settings/configuring-network-settings/configuring-the-hostname-for-your-instance?learn=deploy_an_instance&learnProduct=admin). {% hint style="warning" %} Please refer to the section on using secrets for stack configuration in order to securely store your GitHub @@ -105,11 +107,14 @@ Afterward, you can register a GitLab code repository by running the following CL ```shell zenml code-repository register --type=gitlab \ ---url= --group= --project= \ +--group= --project= \ --token= ``` -where `` is the name of the code repository you are registering, `` is the group of the project, `` is the name of the project, \ is your GitLab Personal Access Token, and \ is the URL of the GitLab instance which defaults to `https://gitlab.com.` You will need to set a URL if you have a self-hosted GitLab instance. +where `` is the name of the code repository you are registering, `` is the group of the project, `` is the name of the project, \ is your GitLab Personal Access Token. + +If you're using a self-hosted GitLab instance, you'll need to also pass the `--instance_url=` and `--host=` options. \ should point to your GitLab instance (defaults to `https://gitlab.com/`) and \ should be the hostname of your GitLab instance (defaults to `gitlab.com`). + {% hint style="warning" %} Please refer to the section on using secrets for stack configuration in order to securely store your GitLab diff --git a/src/zenml/integrations/github/code_repositories/github_code_repository.py b/src/zenml/integrations/github/code_repositories/github_code_repository.py index a26cec87cc3..edd95e0fa03 100644 --- a/src/zenml/integrations/github/code_repositories/github_code_repository.py +++ b/src/zenml/integrations/github/code_repositories/github_code_repository.py @@ -30,6 +30,7 @@ ) from zenml.code_repositories.git import LocalGitRepositoryContext from zenml.logger import get_logger +from zenml.utils import deprecation_utils from zenml.utils.secret_utils import SecretField logger = get_logger(__name__) @@ -39,19 +40,23 @@ class GitHubCodeRepositoryConfig(BaseCodeRepositoryConfig): """Config for GitHub code repositories. Args: - url: The URL of the GitHub instance. + api_url: The GitHub API URL. owner: The owner of the repository. repository: The name of the repository. host: The host of the repository. token: The token to access the repository. """ - url: Optional[str] + api_url: Optional[str] = None owner: str repository: str host: Optional[str] = "github.com" token: Optional[str] = SecretField(default=None) + _deprecation_validator = deprecation_utils.deprecate_pydantic_attributes( + ("url", "api_url") + ) + class GitHubCodeRepository(BaseCodeRepository): """GitHub code repository.""" @@ -120,7 +125,9 @@ def login( RuntimeError: If the login fails. """ try: - self._github_session = Github(self.config.token) + self._github_session = Github( + login_or_token=self.config.token, base_url=self.config.api_url + ) if self.config.token: user = self._github_session.get_user().login logger.debug(f"Logged in as {user}") diff --git a/src/zenml/integrations/gitlab/code_repositories/gitlab_code_repository.py b/src/zenml/integrations/gitlab/code_repositories/gitlab_code_repository.py index 8770bea6e70..f658aa09c1f 100644 --- a/src/zenml/integrations/gitlab/code_repositories/gitlab_code_repository.py +++ b/src/zenml/integrations/gitlab/code_repositories/gitlab_code_repository.py @@ -31,6 +31,7 @@ LocalGitRepositoryContext, ) from zenml.logger import get_logger +from zenml.utils import deprecation_utils from zenml.utils.secret_utils import SecretField logger = get_logger(__name__) @@ -40,19 +41,23 @@ class GitLabCodeRepositoryConfig(BaseCodeRepositoryConfig): """Config for GitLab code repositories. Args: - url: The full URL of the GitLab project. + instance_url: The URL of the GitLab instance. group: The group of the project. project: The name of the GitLab project. host: The host of GitLab in case it is self-hosted instance. token: The token to access the repository. """ - url: Optional[str] + instance_url: Optional[str] = None group: str project: str host: Optional[str] = "gitlab.com" token: str = SecretField() + _deprecation_validator = deprecation_utils.deprecate_pydantic_attributes( + ("url", "instance_url") + ) + class GitLabCodeRepository(BaseCodeRepository): """GitLab code repository.""" @@ -85,7 +90,7 @@ def login(self) -> None: """ try: self._gitlab_session = Gitlab( - f"https://{self.config.host}", private_token=self.config.token + url=self.config.instance_url, private_token=self.config.token ) self._gitlab_session.auth() user = self._gitlab_session.user or None From cc16953746ef43ae3027c02e1ad0d841b9a51445 Mon Sep 17 00:00:00 2001 From: Michael Schuster Date: Wed, 22 Jan 2025 17:57:49 +0100 Subject: [PATCH 6/9] Remove url mention --- .../user-guide/production-guide/connect-code-repository.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/book/user-guide/production-guide/connect-code-repository.md b/docs/book/user-guide/production-guide/connect-code-repository.md index 5e272a6f437..4e896efe66e 100644 --- a/docs/book/user-guide/production-guide/connect-code-repository.md +++ b/docs/book/user-guide/production-guide/connect-code-repository.md @@ -82,12 +82,11 @@ Now, we can install the GitHub integration and register your repository: ```sh zenml integration install github zenml code-repository register --type=github \ ---url=https://github.com/YOUR_USERNAME/YOUR_REPOSITORY_NAME.git \ ---owner=YOUR_USERNAME --repository=YOUR_REPOSITORY_NAME \ ---token=YOUR_GITHUB_PERSONAL_ACCESS_TOKEN +--owner= --repository= \ +--token= ``` -Fill in ``, `YOUR_USERNAME`, `YOUR_REPOSITORY_NAME`, and `YOUR_GITHUB_PERSONAL_ACCESS_TOKEN` with your details. +Fill in ``, ``, ``, and `` with your details. Your code is now connected to your ZenML server. ZenML will automatically detect if your source files are being tracked by GitHub and store the commit hash for each subsequent pipeline run. From d533b56947bbb7e25b935e9677241edddb1daf46 Mon Sep 17 00:00:00 2001 From: Michael Schuster Date: Thu, 23 Jan 2025 14:21:18 +0100 Subject: [PATCH 7/9] Some small doc fixes --- .../connect-your-git-repository.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/book/how-to/project-setup-and-management/setting-up-a-project-repository/connect-your-git-repository.md b/docs/book/how-to/project-setup-and-management/setting-up-a-project-repository/connect-your-git-repository.md index 28aada98648..3699cbb79b0 100644 --- a/docs/book/how-to/project-setup-and-management/setting-up-a-project-repository/connect-your-git-repository.md +++ b/docs/book/how-to/project-setup-and-management/setting-up-a-project-repository/connect-your-git-repository.md @@ -52,9 +52,9 @@ zenml code-repository register --type=github \ --token= ``` -where \ is the name of the code repository you are registering, \ is the owner of the repository, \ is the name of the repository, \ is your GitHub Personal Access Token. +where `` is the name of the code repository you are registering, `` is the owner of the repository, `` is the name of the repository and `` is your GitHub Personal Access Token. -If you're using a self-hosted GitHub Enterprise instance, you'll need to also pass the `--api_url=` and `--host=` options. \ should point to where the GitHub API is reachable (defaults to `https://api.github.com/`) and \ should be the [hostname of your GitHub instance](https://docs.github.com/en/enterprise-server@3.10/admin/configuring-settings/configuring-network-settings/configuring-the-hostname-for-your-instance?learn=deploy_an_instance&learnProduct=admin). +If you're using a self-hosted GitHub Enterprise instance, you'll need to also pass the `--api_url=` and `--host=` options. `` should point to where the GitHub API is reachable (defaults to `https://api.github.com/`) and `` should be the [hostname of your GitHub instance](https://docs.github.com/en/enterprise-server@3.10/admin/configuring-settings/configuring-network-settings/configuring-the-hostname-for-your-instance?learn=deploy_an_instance&learnProduct=admin). {% hint style="warning" %} Please refer to the section on using secrets for stack configuration in order to securely store your GitHub @@ -111,9 +111,9 @@ zenml code-repository register --type=gitlab \ --token= ``` -where `` is the name of the code repository you are registering, `` is the group of the project, `` is the name of the project, \ is your GitLab Personal Access Token. +where `` is the name of the code repository you are registering, `` is the group of the project, `` is the name of the project and `` is your GitLab Personal Access Token. -If you're using a self-hosted GitLab instance, you'll need to also pass the `--instance_url=` and `--host=` options. \ should point to your GitLab instance (defaults to `https://gitlab.com/`) and \ should be the hostname of your GitLab instance (defaults to `gitlab.com`). +If you're using a self-hosted GitLab instance, you'll need to also pass the `--instance_url=` and `--host=` options. `` should point to your GitLab instance (defaults to `https://gitlab.com/`) and `` should be the hostname of your GitLab instance (defaults to `gitlab.com`). {% hint style="warning" %} From bc2b34980a181493be5349e1ed673f5dfc9409eb Mon Sep 17 00:00:00 2001 From: Michael Schuster Date: Thu, 23 Jan 2025 14:28:14 +0100 Subject: [PATCH 8/9] Linting --- .../code_repositories/github_code_repository.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/zenml/integrations/github/code_repositories/github_code_repository.py b/src/zenml/integrations/github/code_repositories/github_code_repository.py index edd95e0fa03..f4c95dba2d9 100644 --- a/src/zenml/integrations/github/code_repositories/github_code_repository.py +++ b/src/zenml/integrations/github/code_repositories/github_code_repository.py @@ -18,7 +18,7 @@ from typing import List, Optional import requests -from github import Github, GithubException +from github import Consts, Github, GithubException from github.Repository import Repository from zenml.code_repositories import ( @@ -100,7 +100,7 @@ def check_github_repo_public(self, owner: str, repo: str) -> None: Raises: RuntimeError: If the repository is not public. """ - url = f"https://api.github.com/repos/{owner}/{repo}" + url = f"{Consts.DEFAULT_BASE_URL}/repos/{owner}/{repo}" response = requests.get(url, timeout=7) try: @@ -108,12 +108,15 @@ def check_github_repo_public(self, owner: str, repo: str) -> None: pass else: raise RuntimeError( - "It is not possible to access this repository as it does not appear to be public." - "Access to private repositories is only possible when a token is provided. Please provide a token and try again" + "It is not possible to access this repository as it does " + "not appear to be public. Access to private repositories " + "is only possible when a token is provided. Please provide " + "a token and try again" ) except Exception as e: raise RuntimeError( - f"An error occurred while checking if repository is public: {str(e)}" + "An error occurred while checking if repository is public: " + f"{str(e)}" ) def login( @@ -126,7 +129,8 @@ def login( """ try: self._github_session = Github( - login_or_token=self.config.token, base_url=self.config.api_url + login_or_token=self.config.token, + base_url=self.config.api_url or Consts.DEFAULT_BASE_URL, ) if self.config.token: user = self._github_session.get_user().login From 594a652245bb5fe377371f51dde86b0b593ea6fc Mon Sep 17 00:00:00 2001 From: Michael Schuster Date: Thu, 23 Jan 2025 15:46:57 +0100 Subject: [PATCH 9/9] Fix unit test --- tests/unit/pipelines/test_build_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/pipelines/test_build_utils.py b/tests/unit/pipelines/test_build_utils.py index de278fac778..b8b7e86eb20 100644 --- a/tests/unit/pipelines/test_build_utils.py +++ b/tests/unit/pipelines/test_build_utils.py @@ -193,7 +193,6 @@ def test_build_uses_correct_settings(mocker, empty_pipeline): # noqa: F811 entrypoint=build_config.entrypoint, extra_files=build_config.extra_files, include_files=True, - download_files=False, code_repository=None, ) assert build.pipeline.id == pipeline_id