diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 46541f0..b234146 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ "3.8", "3.9" ] + python-version: [ "3.9", "3.10", "3.11" ] steps: - uses: actions/checkout@v3 @@ -21,10 +21,12 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + version: 2.1.1 - name: Install library run: poetry install --no-interaction - - name: Lint and test with poetry + - name: Lint with poetry + run: poetry run pylint foliant + - name: Test with poetry run: | poetry run pytest --cov=foliant - poetry run codecov - poetry run pylint foliant \ No newline at end of file + poetry run codecov \ No newline at end of file diff --git a/changelog.md b/changelog.md index 4039e7e..58be905 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,8 @@ +# 1.0.14 + +- Add the `--only-partial` argument for to limit the list of files that the foliant must process. + This function allows you to build a website from a single file, a list of files, or a glob mask. + # 1.0.13 - Add the `clean_registry` function to `make`. diff --git a/foliant/__init__.py b/foliant/__init__.py index bf66ffc..cc08f08 100644 --- a/foliant/__init__.py +++ b/foliant/__init__.py @@ -1 +1 @@ -__version__ = '1.0.13' +__version__ = '1.0.14' diff --git a/foliant/backends/base.py b/foliant/backends/base.py index a42a501..dd03f74 100644 --- a/foliant/backends/base.py +++ b/foliant/backends/base.py @@ -1,11 +1,13 @@ -from importlib import import_module -from shutil import copytree from datetime import date +from importlib import import_module from logging import Logger +from pathlib import Path +from shutil import copytree +from typing import Union, List +from foliant.partial_copy import PartialCopy from foliant.utils import spinner - class BaseBackend(): '''Base backend. All backends must inherit from this one.''' @@ -50,6 +52,8 @@ def apply_preprocessor(self, preprocessor: str or dict): :param preprocessor: Preprocessor name or a dict of the preprocessor name and its options ''' + preprocessor_name = None + preprocessor_options = {} if isinstance(preprocessor, str): preprocessor_name, preprocessor_options = preprocessor, {} @@ -82,6 +86,17 @@ def apply_preprocessor(self, preprocessor: str or dict): f'Failed to apply preprocessor {preprocessor_name}: {exception}' ) from exception + @staticmethod + def partial_copy( + source: Union[str, Path, List[Union[str, Path]]], + root: Union[str, Path], + destination: Union[str, Path], + ) -> None: + """ + Delegates to the PartialCopy class for file copying operations. + """ + PartialCopy.partial_copy(source, root, destination) + def preprocess_and_make(self, target: str) -> str: '''Apply preprocessors required by the selected backend and defined in the config file, then run the ``make`` method. @@ -92,8 +107,10 @@ def preprocess_and_make(self, target: str) -> str: ''' src_path = self.project_path / self.config['src_dir'] - - copytree(src_path, self.working_dir) + if self.context['only_partial'] != "": + self.partial_copy(self.context['only_partial'], src_path, self.working_dir) + else: + copytree(src_path, self.working_dir) common_preprocessors = ( *self.required_preprocessors_before, diff --git a/foliant/cli/__init__.py b/foliant/cli/__init__.py index 7bb8af8..0b112cf 100644 --- a/foliant/cli/__init__.py +++ b/foliant/cli/__init__.py @@ -12,7 +12,6 @@ class Foliant(*get_available_clis().values()): @set_help({'version': 'show version and exit'}) def _root(self, version=False): - # pylint: disable=no-self-use if version: print(f'Foliant v.{foliant_version}') diff --git a/foliant/cli/make.py b/foliant/cli/make.py index 9478292..581d97b 100644 --- a/foliant/cli/make.py +++ b/foliant/cli/make.py @@ -4,6 +4,7 @@ from importlib import import_module from logging import DEBUG, WARNING from typing import List, Dict, Tuple +from glob import glob from cliar import set_arg_map, set_metavars, set_help, ignore from prompt_toolkit import prompt @@ -87,6 +88,27 @@ def get_matching_backend(target: str, available_backends: Dict[str, Tuple[str]]) except KeyboardInterrupt as kbd_interrupt: raise BackendError('No backend specified') from kbd_interrupt + @staticmethod + def prepare_list_of_file(only_partial, project_path): + list_of_files = [] + if isinstance(only_partial, str): + if ',' in only_partial: + only_partial = [item.strip() for item in only_partial.split(',')] + elif any(char in only_partial for char in '*?['): + list_of_files = [Path(file) for file in glob(only_partial, recursive=True)] + else: + only_partial = [Path(only_partial.strip())] + + if isinstance(only_partial, list): + for item in only_partial: + item_path = Path(item) if Path(item).is_absolute() else Path(project_path, item) + list_of_files.append(item_path) + elif isinstance(only_partial, (str, Path)): + only_partial_path = Path(only_partial) if isinstance(only_partial, + str) else only_partial + list_of_files.append(only_partial_path) + return list_of_files + def clean_registry(self, project_path): multiprojectcache_dir = os.path.join(project_path, '.multiprojectcache') if os.path.isdir(multiprojectcache_dir): @@ -140,6 +162,7 @@ def get_config( 'logs_dir': 'Path to the directory to store logs, defaults to project path.', 'quiet': 'Hide all output accept for the result. Useful for piping.', 'keep_tmp': 'Keep the tmp directory after the build.', + 'only_partial': 'using only a partial list of files', 'debug': 'Log all events during build. If not set, only warnings and errors are logged.' } ) @@ -152,6 +175,7 @@ def make( logs_dir='', quiet=False, keep_tmp=False, + only_partial='', debug=False ): '''Make TARGET with BACKEND.''' @@ -161,6 +185,7 @@ def make( # pylint: disable=consider-using-sys-exit self.logger.setLevel(DEBUG if debug else WARNING) + result = None if logs_dir: super().__init__(logs_dir) @@ -173,6 +198,9 @@ def make( self.clean_registry(project_path) + if only_partial != "": + only_partial = self.prepare_list_of_file(only_partial, project_path) + try: if backend: self.validate_backend(backend, target) @@ -189,7 +217,9 @@ def make( 'project_path': project_path, 'config': config, 'target': target, - 'backend': backend + 'backend': backend, + 'keep_tmp': keep_tmp, + 'only_partial': only_partial } backend_module = import_module(f'foliant.backends.{backend}') diff --git a/foliant/partial_copy.py b/foliant/partial_copy.py new file mode 100644 index 0000000..7c2681c --- /dev/null +++ b/foliant/partial_copy.py @@ -0,0 +1,276 @@ +import os +import re +from pathlib import Path +from shutil import copy +from typing import Union, List, Set +import frontmatter + + +class PartialCopy: + """ + Handles partial copying of files with dependency tracking and content modification. + """ + + @staticmethod + def partial_copy( + source: Union[str, Path, List[Union[str, Path]]], + root: Union[str, Path], + destination: Union[str, Path], + ) -> None: + """ + Copies files, a list of files, + or files matching a glob pattern to the specified folder. + Creates all necessary directories if they don't exist. + """ + + if isinstance(source, list): + source_display = "\n- " + "\n- ".join([str(item) for item in source]) + else: + source_display = str(source) + + print(f"Partial build is processing...\nList of files: {source_display}") + + destination_path = Path(destination) + root_path = Path(root) + image_extensions = {'.jpg', '.jpeg', '.png', '.svg', '.gif', '.bmp', '.webp'} + image_pattern = re.compile( + r'!\[.*?\]\((.*?)( \"(.+)\"|)\)|.*?)(\")|)' + r'(?:\s[^\<\>]*)?\>(?P.*?)\<\/(?:include)\>', + flags=re.DOTALL + ) + + copied_files_count = 0 + processed_files = set() + max_recursion_depth = 10 + + def _modify_markdown_file( # pylint: disable=too-many-arguments + file_path: Union[str, Path], + dst_file_path: Union[str, Path], + not_build: bool = True, + remove_content: bool = True, + keep_first_header: bool = True, + create_frontmatter: bool = True, + dry_run: bool = False, + ): + """Modify a Markdown file's frontmatter and content.""" + try: + file_path = Path(file_path) + content = file_path.read_text(encoding='utf-8') + post = frontmatter.loads(content) + changes_made = False + + # Process modifications + changes_made = PartialCopy._process_frontmatter(post, not_build) or changes_made + changes_made = PartialCopy._process_content( + post, remove_content, keep_first_header + ) or changes_made + + if not PartialCopy._has_frontmatter(post) and create_frontmatter and ( + not_build is not None or changes_made + ): + changes_made = True + + if not changes_made: + return False, content + + output = PartialCopy._serialize_output(post, create_frontmatter) + + if dry_run: + return True, output + + dst_file_path.write_text(output, encoding='utf-8') + return True, output + + except Exception as e: # pylint: disable=broad-exception-caught + print(f"ERROR: processing {file_path}: {str(e)}") + return False, content + + def _find_referenced_images(file_path: Path) -> Set[Path]: + """Finds all image files referenced in the given file.""" + image_paths = set() + with open(file_path, 'r', encoding='utf-8') as file: + content = file.read() + for match in image_pattern.findall(content): + for group in match: + if group: + image_path = Path(file_path).parent / Path(group) + if image_path.suffix.lower() in image_extensions: + image_paths.add(image_path) + return image_paths + + def _copy_files_without_content(src_dir: str, dst_dir: str): + """Copies files, leaving only the first-level header.""" + if not os.path.exists(dst_dir): + os.makedirs(dst_dir) + + for file_root, _, files in os.walk(src_dir): + for file_name in files: + src_file_path = os.path.join(file_root, file_name) + dirs = os.path.relpath(file_root, src_dir) + dst_file_path = Path(os.path.join(dst_dir, dirs, file_name)) + dst_file_path.parent.mkdir(parents=True, exist_ok=True) + if file_name.endswith('.md'): + _modify_markdown_file(src_file_path, dst_file_path) + + def _prepare_paths_list(file_path, relative_path_root) -> List: + include_paths = [] + match_includes = re.finditer(include_statement_pattern, + file_path.read_text(encoding='utf-8')) + for path in match_includes: + paths_list = [] + groups = path.groupdict() + if groups["path"]: + paths_list.append(groups["path"]) + if groups["src"]: + paths_list.append(groups["src"]) + + for p in paths_list: + _path = Path(p) + if isinstance(file_path, Path): + rel_path = file_path.parent / _path + if rel_path.exists(): + _path = rel_path + if not _path.exists(): + _path = relative_path_root / _path + if _path.exists(): + include_paths.append(_path) + return include_paths + + def _copy_files_recursive(files_to_copy: List, recursion_level: int = 0): # pylint: disable=too-many-branches + """Recursively copies files and their dependencies.""" + nonlocal copied_files_count + + if recursion_level > max_recursion_depth: + print(f"WARNING: Maximum recursion depth ({max_recursion_depth}) exceeded") + return + + referenced_images = set() + + for file_path in files_to_copy: + file_path = Path(file_path) + + # Check if this file was already processed + if file_path in processed_files: + continue + processed_files.add(file_path) + + if not file_path.exists(): + print(f"WARNING: File {file_path} does not exist, skipping") + continue + + if file_path.is_relative_to(root_path): + relative_path = file_path.relative_to(root_path) + destination_file_path = destination_path / relative_path + destination_file_path.parent.mkdir(parents=True, exist_ok=True) + + if file_path.exists(): + # Find and copy includes + include_paths = _prepare_paths_list(file_path, root_path) + _copy_files_recursive(include_paths, recursion_level + 1) + + # Find referenced images + referenced_images.update(_find_referenced_images(file_path)) + + # Copy the file + try: + copy(file_path, destination_file_path) + copied_files_count += 1 + except FileNotFoundError as e: + print(f"ERROR: File not found: {e}") + except Exception as e: # pylint: disable=broad-exception-caught + print(f"ERROR: copying {file_path}: {e}") + + # Copy referenced images + for image_path in referenced_images: + image_path = Path(image_path) + if image_path in processed_files: + continue + processed_files.add(image_path) + + if image_path.exists(): + if image_path.is_relative_to(root_path): + src_image_path = image_path.relative_to(root_path) + else: + src_image_path = image_path + + dst_image_path = destination_path / src_image_path + dst_image_path.parent.mkdir(parents=True, exist_ok=True) + + if image_path != dst_image_path: + try: + copy(image_path, dst_image_path) + copied_files_count += 1 + except Exception as e: # pylint: disable=broad-exception-caught + print(f"ERROR: copying image {image_path}: {e}") + + # Main logic with verification + try: + # Normalize source to file list + if isinstance(source, (str, Path)): + source_path = Path(source) + if source_path.is_dir(): + files_to_copy = list(source_path.rglob('*')) + elif '*' in str(source_path): + # Handle glob pattern + files_to_copy = list(root_path.glob(str(source_path))) + else: + files_to_copy = [source_path] + else: + files_to_copy = [Path(f) for f in source] + + print(f"Total files to process: {len(files_to_copy)}") + + # Copy files + _copy_files_without_content(root_path, destination_path) + _copy_files_recursive(files_to_copy) + + if copied_files_count == 0: + print("WARNING: No files were copied!") + elif copied_files_count < len(files_to_copy): + print(f"WARNING: Only {copied_files_count} out of {len(files_to_copy)} files were copied") # pylint: disable=line-too-long + + except Exception as e: # pylint: disable=broad-exception-caught + print(f"ERROR: during copy operation: {e}") + raise + + @staticmethod + def _process_frontmatter(post, not_build: bool) -> bool: + """Process frontmatter modifications.""" + if not_build is not None and post.get('not_build') != not_build: + post['not_build'] = not_build + return True + return False + + @staticmethod + def _process_content(post, remove_content: bool, keep_first_header: bool) -> bool: + """Process content modifications.""" + if remove_content and post.content.strip(): + new_content = PartialCopy._extract_first_header( + post.content) if keep_first_header else '' + if post.content != new_content: + post.content = new_content + return True + return False + + @staticmethod + def _extract_first_header(content: str) -> str: + """Extract first H1 header from content.""" + h1_match = re.search(r'^#\s+.+$', content, flags=re.MULTILINE) + return h1_match.group(0) + '\n' if h1_match else '' + + @staticmethod + def _serialize_output(post, create_frontmatter: bool) -> str: + """Serialize post to text with proper formatting.""" + output = frontmatter.dumps(post) + if not PartialCopy._has_frontmatter(post) and create_frontmatter: + output = f"---\n{output}" + return output + '\n' + + @staticmethod + def _has_frontmatter(post: frontmatter.Post) -> bool: + """Check if post has existing frontmatter.""" + return hasattr(post, 'metadata') and (post.metadata or hasattr(post, 'fm')) diff --git a/foliant/preprocessors/base.py b/foliant/preprocessors/base.py index 888e4d0..f380191 100644 --- a/foliant/preprocessors/base.py +++ b/foliant/preprocessors/base.py @@ -38,6 +38,7 @@ def get_options(options_string: str) -> Dict[str, OptionValue]: def __init__(self, context: dict, logger: Logger, quiet=False, debug=False, options={}): # pylint: disable=dangerous-default-value # pylint: disable=too-many-arguments + # pylint: disable=duplicate-code self.project_path = context['project_path'] self.config = context['config'] diff --git a/foliant/utils.py b/foliant/utils.py index a2d8803..ecda597 100644 --- a/foliant/utils.py +++ b/foliant/utils.py @@ -1,14 +1,14 @@ '''Various utilities used here and there in the Foliant code.''' from contextlib import contextmanager -from pkgutil import iter_modules from importlib import import_module +from importlib.metadata import distributions +from logging import Logger +from pathlib import Path +from pkgutil import iter_modules from shutil import rmtree from traceback import format_exc -from pathlib import Path -from logging import Logger -from typing import List, Dict, Tuple, Type, Set -import pkg_resources +from typing import Dict, List, Set, Tuple, Type def get_available_tags() -> Set[str]: @@ -26,7 +26,9 @@ def get_available_tags() -> Set[str]: if modname == 'base': continue - result.update(importer.find_module(modname).load_module(modname).Preprocessor.tags) + spec = importer.find_spec(modname) + module = spec.loader.load_module() + result.update(module.Preprocessor.tags) return result @@ -49,7 +51,9 @@ def get_available_config_parsers() -> Dict[str, Type]: if modname == 'base': continue - result[modname] = importer.find_module(modname).load_module(modname).Parser + spec = importer.find_spec(modname) + module = spec.loader.load_module() + result[modname] = module.Parser return result @@ -72,7 +76,9 @@ def get_available_clis() -> Dict[str, Type]: if modname == 'base': continue - result[modname] = importer.find_module(modname).load_module(modname).Cli + spec = importer.find_spec(modname) + module = spec.loader.load_module() + result[modname] = module.Cli return result @@ -96,7 +102,9 @@ def get_available_backends() -> Dict[str, Tuple[str]]: if modname == 'base': continue - result[modname] = importer.find_module(modname).load_module(modname).Backend.targets + spec = importer.find_spec(modname) + module = spec.loader.load_module() + result[modname] = module.Backend.targets return result @@ -110,15 +118,16 @@ def get_foliant_packages() -> List[str]: # pylint: disable=not-an-iterable foliant_packages = [] - all_packages = pkg_resources.working_set + all_packages = distributions() for package in all_packages: - if package.key == 'foliant': + foliant_core_version = None + if package.metadata["Name"] == 'foliant': foliant_core_version = package.version - elif package.key.startswith('foliantcontrib.'): + elif package.metadata["Name"].startswith('foliantcontrib.'): foliant_packages.append( - f'{package.key.replace("foliantcontrib.", "", 1)} {package.version}' + f'{package.metadata["Name"].replace("foliantcontrib.", "", 1)} {package.version}' ) foliant_packages = sorted(foliant_packages) diff --git a/pylintrc b/pylintrc index 79bde8c..1f3901d 100644 --- a/pylintrc +++ b/pylintrc @@ -55,88 +55,11 @@ confidence= # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" disable=missing-docstring, - print-statement, - parameter-unpacking, - unpacking-in-except, - old-raise-syntax, - backtick, - long-suffix, - old-ne-operator, - old-octal-literal, - import-star-module-level, - non-ascii-bytes-literal, - invalid-unicode-literal, - raw-checker-failed, - bad-inline-option, - locally-disabled, - locally-enabled, - file-ignored, - suppressed-message, - useless-suppression, - deprecated-pragma, - too-few-public-methods, - apply-builtin, - basestring-builtin, - buffer-builtin, - cmp-builtin, - coerce-builtin, - execfile-builtin, - file-builtin, - long-builtin, - raw_input-builtin, - reduce-builtin, - standarderror-builtin, - unicode-builtin, - xrange-builtin, - coerce-method, - delslice-method, - getslice-method, - setslice-method, - no-absolute-import, - old-division, - dict-iter-method, - dict-view-method, - next-method-called, - metaclass-assignment, - indexing-exception, - raising-string, - reload-builtin, - oct-method, - hex-method, - nonzero-method, - cmp-method, - input-builtin, - round-builtin, - intern-builtin, - unichr-builtin, - map-builtin-not-iterating, - zip-builtin-not-iterating, - range-builtin-not-iterating, - filter-builtin-not-iterating, - using-cmp-argument, - eq-without-hash, - div-method, - idiv-method, - rdiv-method, - exception-message-attribute, - invalid-str-codec, - sys-max-int, - bad-python3-import, - deprecated-string-function, - deprecated-str-translate-call, - deprecated-itertools-function, - deprecated-types-field, - next-method-defined, - dict-items-not-iterating, - dict-keys-not-iterating, - dict-values-not-iterating, - deprecated-operator-function, - deprecated-urllib-function, - xreadlines-attribute, - deprecated-sys-function, - exception-escape, - comprehension-escape, - duplicate-code + too-many-locals, + too-few-public-methods, + too-many-statements, + unreachable, + too-many-positional-arguments # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option @@ -314,8 +237,8 @@ max-module-lines=1000 # separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. # `trailing-comma` allows a space between comma and closing bracket: (a, ). # `empty-line` allows space-only lines. -no-space-check=trailing-comma, - dict-separator +# no-space-check=trailing-comma, +# dict-separator # Allow the body of a class to be on the same line as the declaration if body # contains single statement. @@ -546,4 +469,4 @@ known-third-party=enchant # Exceptions that will emit a warning when being caught. Defaults to # "Exception" -overgeneral-exceptions=Exception +overgeneral-exceptions=builtins.Exception diff --git a/pyproject.toml b/pyproject.toml index 96a7fdd..ac659f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "foliant" -version = "1.0.13" +version = "1.0.14" description = "Modular, Markdown-based documentation generator that makes pdf, docx, html, and more." license = "MIT" authors = ["Konstantin Molchanov "] @@ -11,17 +11,18 @@ documentation = "https://foliant-docs.github.io/docs/" keywords = ["documentation", "markdown", "html", "docx", "pdf"] [tool.poetry.dependencies] -python = "^3.6" -pyyaml = "^5.1.1" -cliar = "^1.3.2" -prompt_toolkit = "^2.0" +python = "^3.9" +pyyaml = "6.0.2" +cliar = "^1.3.5" +prompt_toolkit = "^3.0.50" +python-frontmatter = ">=1.1.0" -[tool.poetry.dev-dependencies] -pytest = "^3.6" -pytest-datadir = "^1.0" -pylint = "^2.5" -pytest-cov = "^2.5" -codecov = "^2.0" +[tool.poetry.group.dev.dependencies] +pytest = "^8.3.5" +pytest-datadir = "^1.6.1" +pylint = "^3.3.6" +pytest-cov = "^6.0.0" +codecov = "^2.1.13" [tool.poetry.scripts] foliant = 'foliant.cli:entry_point' diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..1c9eb3a --- /dev/null +++ b/test.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# before testing make sure that you have installed the fresh version of preprocessor: +poetry install --no-interaction + +# run tests +poetry lock +poetry update +poetry run pylint foliant +poetry run pytest --cov=foliant && \ +poetry run codecov diff --git a/test_in_docker.sh b/test_in_docker.sh new file mode 100755 index 0000000..562bb56 --- /dev/null +++ b/test_in_docker.sh @@ -0,0 +1,21 @@ +#!/bin/bash + + +PYTHON_VERSIONS=("3.9" "3.10" "3.11") + +for version in "${PYTHON_VERSIONS[@]}"; do + # Write Dockerfile + echo "FROM python:${version}-alpine3.20" > Dockerfile + echo "RUN apk add --no-cache --upgrade bash && \ + pip install poetry==2.1.1 && \ + apk add git" >> Dockerfile + echo "RUN git config --global --add safe.directory /app" >> Dockerfile + + # Run tests in docker + docker build . -t test-foliant:latest + + docker run --rm -it -v "./:/app/" -w /app/ test-foliant:latest "./test.sh" + + # Remove Dockerfile + rm Dockerfile +done diff --git a/tests/test_backends.py b/tests/test_backends.py new file mode 100644 index 0000000..ad03940 --- /dev/null +++ b/tests/test_backends.py @@ -0,0 +1,195 @@ +from unittest import TestCase +from pathlib import Path +from tempfile import TemporaryDirectory + +from foliant.backends.base import BaseBackend +from foliant.cli.make import Cli + +class TestBackendCopyFiles(TestCase): + def setUp(self): + # Create a temporary directory for testing + self.test_dir = TemporaryDirectory() + self.source_dir = Path(self.test_dir.name) / "source" + self.destination_dir = Path(self.test_dir.name) / "destination" + self.working_dir = self.destination_dir + self.source_dir.mkdir() + self.destination_dir.mkdir() + + # Create some test files + (self.source_dir / "file1.txt").write_text("Hello, file1!") + (self.source_dir / "file2.txt").write_text("Hello, file2!") + (self.source_dir / "subfolder").mkdir() + (self.source_dir / "subfolder" / "file3.txt").write_text("Hello, file3!") + (self.source_dir / "file2.md").write_text("# Header\nSome content") + (self.source_dir / "subfolder" / "file3.md").write_text("# Another Header\nMore content") + + def tearDtown(self): + # Clean up the temporary directory + self.test_dir.cleanup() + + def test_copy_single_file(self): + # Test copying a single file + source_file = self.source_dir / "file1.txt" + source_file = Cli.prepare_list_of_file(source_file, self.destination_dir) + BaseBackend.partial_copy(source_file, self.source_dir, self.working_dir) + + # Check if the file was copied + self.assertTrue((self.destination_dir / "file1.txt").exists()) + self.assertEqual((self.destination_dir / "file1.txt").read_text(), "Hello, file1!") + + def test_copy_list_of_files(self): + # Test copying a list of files + source_files = [ + self.source_dir / "file1.txt", + self.source_dir / "file2.txt" + ] + + source_files = Cli.prepare_list_of_file(source_files, self.destination_dir) + BaseBackend.partial_copy(source_files, self.source_dir, self.working_dir) + + # Check if the files were copied + self.assertTrue((self.destination_dir / "file1.txt").exists()) + self.assertTrue((self.destination_dir / "file2.txt").exists()) + + + def test_copy_list_of_files_two(self): + # Test copying a list of files + source_files = [ + str(self.source_dir / "file1.txt"), + str(self.source_dir / "file2.md") + ] + source_files = Cli.prepare_list_of_file(source_files, self.destination_dir) + BaseBackend.partial_copy(source_files, self.source_dir, self.working_dir) + + # Check if the files were copied + self.assertTrue((self.destination_dir / "file1.txt").exists()) + self.assertTrue((self.destination_dir / "file2.md").exists()) + + def test_copy_glob_pattern(self): + # Test copying files matching a glob pattern + glob_pattern = str(self.source_dir / "*.txt") + glob_pattern = Cli.prepare_list_of_file(glob_pattern, self.destination_dir) + BaseBackend.partial_copy(glob_pattern, self.source_dir, self.working_dir) + + # Check if the files were copied + self.assertTrue((self.destination_dir / "file1.txt").exists()) + self.assertTrue((self.destination_dir / "file2.txt").exists()) + self.assertFalse((self.destination_dir / "file3.txt").exists()) # file3 is in a subfolder + + def test_copy_glob_pattern_md(self): + # Test copying files matching a glob pattern + glob_pattern = str(self.source_dir / '*2.md') + glob_pattern = Cli.prepare_list_of_file(glob_pattern, self.destination_dir) + BaseBackend.partial_copy(glob_pattern, self.source_dir, self.working_dir) + + # Check if the files were copied + self.assertTrue((self.destination_dir / "file2.md").exists()) + self.assertEqual((self.destination_dir / "file2.md").read_text(), "# Header\nSome content") + self.assertTrue((self.destination_dir / "subfolder" / "file3.md").exists()) + self.assertEqual((self.destination_dir / "subfolder" / "file3.md").read_text(), "---\nnot_build: true\n---\n\n# Another Header\n") # Only '*2.md' files should be copied with content + + def test_copy_glob_pattern_recursive(self): + # Test copying files matching a recursive glob pattern + glob_pattern = str(self.source_dir / "**" / "*.txt") + glob_pattern = Cli.prepare_list_of_file(glob_pattern, self.destination_dir) + BaseBackend.partial_copy(glob_pattern, self.source_dir, self.working_dir) + + # Check if the files were copied, including the one in the subfolder + self.assertTrue((self.destination_dir / "file1.txt").exists()) + self.assertTrue((self.destination_dir / "file2.txt").exists()) + self.assertTrue((self.destination_dir / "subfolder" / "file3.txt").exists()) + + def test_copy_directory_structure(self): + # Test copying a file while preserving directory structure + source_file = self.source_dir / "subfolder" / "file3.txt" + source_file = Cli.prepare_list_of_file(source_file, self.destination_dir) + BaseBackend.partial_copy(source_file, self.source_dir, self.working_dir) + + # Check if the file was copied with the directory structure + self.assertTrue((self.destination_dir / "subfolder" / "file3.txt").exists()) + + # def test_copy_nonexistent_file(self): + # # Test copying a nonexistent file (should raise FileNotFoundError) + # source_file = self.source_dir / "nonexistent.txt" + # with self.assertRaises(FileNotFoundError): + # BaseBackend.partial_copy(source_file, self.source_dir, self.working_dir) + + # def test_copy_nonexistent_glob(self): + # # Test copying with a glob pattern that matches no files + # glob_pattern = str(self.source_dir / "*_suffix.md") + # BaseBackend.partial_copy(glob_pattern, self.source_dir, self.working_dir) + + # # Check that no files were copied + # self.assertTrue((self.destination_dir / "file2.md").exists()) + # self.assertEqual((self.destination_dir / "file2.md").read_text(), "# Header\n") + # self.assertTrue((self.destination_dir / "subfolder" / "file3.md").exists()) + # self.assertEqual((self.destination_dir / "subfolder" / "file3.md").read_text(), "# Another Header\n") + + # def test_copy_to_nonexistent_destination(self): + # # Test copying to a nonexistent destination (should create the destination folder) + # new_destination = self.destination_dir / "new_folder" + # BaseBackend.partial_copy(self.source_dir / "file1.txt", new_destination, self.source_dir, self.working_dir) + + # # Check if the file was copied and the destination folder was created + # self.assertTrue(new_destination.exists()) + # self.assertTrue((new_destination / "file1.txt").exists()) + + def test_copy_path_object(self): + # Test copying using Path objects + source_file = self.source_dir / "file1.txt" + destination = self.destination_dir / "file1.txt" + source_file = Cli.prepare_list_of_file(source_file, destination) + BaseBackend.partial_copy(source_file, self.source_dir, self.working_dir) + + # Check if the file was copied + self.assertTrue(destination.exists()) + + def test_copy_text_file(self): + source_file = Cli.prepare_list_of_file(str(self.source_dir / "file1.txt"), self.destination_dir) + # Test copying a text file + source_file = Cli.prepare_list_of_file(str(self.source_dir / "file1.txt"), self.destination_dir) + BaseBackend.partial_copy(source_file, self.source_dir, self.working_dir) + + # Check if the file was copied + self.assertTrue((self.destination_dir / "file1.txt").exists()) + self.assertEqual((self.destination_dir / "file1.txt").read_text(), "Hello, file1!") + + def test_copy_markdown_file(self): + source_file = Cli.prepare_list_of_file(str(self.source_dir / "file2.md"), self.destination_dir) + # Test copying a Markdown file + BaseBackend.partial_copy(source_file, self.source_dir, self.working_dir) + + # Check if only the header was copied + self.assertTrue((self.destination_dir / "file2.md").exists()) + self.assertEqual((self.destination_dir / "file2.md").read_text(), "# Header\nSome content") + + def test_copy_directory_structure_with_header(self): + source_file = Cli.prepare_list_of_file(str(self.source_dir / "subfolder" / "file3.md"), self.destination_dir) + # Test copying a file while preserving directory structure + BaseBackend.partial_copy(source_file, self.source_dir, self.working_dir) + + # Check if the file was copied with the directory structure + self.assertTrue((self.destination_dir / "subfolder" / "file3.md").exists()) + self.assertEqual((self.destination_dir / "subfolder" / "file3.md").read_text(), "# Another Header\nMore content") + + def test_copy_referenced_images(self): + # Create a Markdown file with image references + md_content = "# Header\n![Image 1](images/image1.png)\n![Image 2](images/image2.jpg)" + (self.source_dir / "file1.md").write_text(md_content) + + # Create referenced images + (self.source_dir / "images").mkdir() + (self.source_dir / "images" / "image1.png").write_text("Fake PNG content") + (self.source_dir / "images" / "image2.jpg").write_text("Fake JPG content") + + source_file = Cli.prepare_list_of_file(str(self.source_dir / "file1.md"), self.destination_dir) + + # Copy files + BaseBackend.partial_copy(source_file, self.source_dir, self.working_dir) + + # Check if the Markdown file was copied + self.assertTrue((self.destination_dir / "file1.md").exists()) + + # Check if referenced images were copied + self.assertTrue((self.destination_dir / "images" / "image1.png").exists()) + self.assertTrue((self.destination_dir / "images" / "image2.jpg").exists()) diff --git a/tests/test_partial_copy.py b/tests/test_partial_copy.py new file mode 100644 index 0000000..64d9e3b --- /dev/null +++ b/tests/test_partial_copy.py @@ -0,0 +1,246 @@ +import frontmatter + +from unittest import TestCase +from pathlib import Path +from tempfile import TemporaryDirectory +from foliant.partial_copy import PartialCopy +from foliant.cli.make import Cli + + +class TestPartialCopy(TestCase): + def setUp(self): + # Create a temporary directory for testing + self.test_dir = TemporaryDirectory() + self.source_dir = Path(self.test_dir.name) / "source" + self.destination_dir = Path(self.test_dir.name) / "destination" + self.source_dir.mkdir() + self.destination_dir.mkdir() + + # Create some test files + (self.source_dir / "file1.txt").write_text("Hello, file1!") + (self.source_dir / "file2.txt").write_text("Hello, file2!") + (self.source_dir / "subfolder").mkdir() + (self.source_dir / "subfolder" / "file3.txt").write_text("Hello, file3!") + (self.source_dir / "file2.md").write_text("# Header\nSome content") + (self.source_dir / "subfolder" / "file3.md").write_text("# Another Header\nMore content") + (self.source_dir / "file4.md").write_text("---\nkey: value\n---\n# Header\nSome content") + + def tearDown(self): + # Clean up the temporary directory + self.test_dir.cleanup() + + def test_copy_single_file(self): + # Test copying a single file + source_file = self.source_dir / "file1.txt" + PartialCopy.partial_copy(source_file, self.source_dir, self.destination_dir) + + # Check if the file was copied + self.assertTrue((self.destination_dir / "file1.txt").exists()) + self.assertEqual((self.destination_dir / "file1.txt").read_text(), "Hello, file1!") + + def test_copy_list_of_files(self): + # Test copying a list of files + source_files = [ + self.source_dir / "file1.txt", + self.source_dir / "file2.txt" + ] + + PartialCopy.partial_copy(source_files, self.source_dir, self.destination_dir) + + # Check if the files were copied + self.assertTrue((self.destination_dir / "file1.txt").exists()) + self.assertTrue((self.destination_dir / "file2.txt").exists()) + + def test_copy_list_of_files_two(self): + # Test copying a list of files + source_files = [ + str(self.source_dir / "file1.txt"), + str(self.source_dir / "file2.md") + ] + PartialCopy.partial_copy(source_files, self.source_dir, self.destination_dir) + + # Check if the files were copied + self.assertTrue((self.destination_dir / "file1.txt").exists()) + self.assertTrue((self.destination_dir / "file2.md").exists()) + + def test_copy_glob_pattern(self): + # Test copying files matching a glob pattern + glob_pattern = str(self.source_dir / "*.txt") + glob_pattern = Cli.prepare_list_of_file(glob_pattern, self.destination_dir) + PartialCopy.partial_copy(glob_pattern, self.source_dir, self.destination_dir) + + # Check if the files were copied + self.assertTrue((self.destination_dir / "file1.txt").exists()) + self.assertTrue((self.destination_dir / "file2.txt").exists()) + self.assertFalse((self.destination_dir / "file3.txt").exists()) # file3 is in a subfolder + + def test_copy_glob_pattern_md(self): + # Test copying files matching a glob pattern + glob_pattern = str(self.source_dir / '*2.md') + glob_pattern = Cli.prepare_list_of_file(glob_pattern, self.destination_dir) + PartialCopy.partial_copy(glob_pattern, self.source_dir, self.destination_dir) + + # Check if the files were copied + self.assertTrue((self.destination_dir / "file2.md").exists()) + # Markdown files should be processed (content removed, frontmatter added) + content = (self.destination_dir / "file2.md").read_text() + self.assertIn("# Header\nSome content", content) + + def test_copy_glob_pattern_recursive(self): + # Test copying files matching a recursive glob pattern + glob_pattern = str(self.source_dir / "**" / "*.txt") + glob_pattern = Cli.prepare_list_of_file(glob_pattern, self.destination_dir) + PartialCopy.partial_copy(glob_pattern, self.source_dir, self.destination_dir) + + # Check if the files were copied, including the one in the subfolder + self.assertTrue((self.destination_dir / "file1.txt").exists()) + self.assertTrue((self.destination_dir / "file2.txt").exists()) + self.assertTrue((self.destination_dir / "subfolder" / "file3.txt").exists()) + + def test_copy_directory_structure(self): + # Test copying a file while preserving directory structure + source_file = self.source_dir / "subfolder" / "file3.txt" + PartialCopy.partial_copy(source_file, self.source_dir, self.destination_dir) + + # Check if the file was copied with the directory structure + self.assertTrue((self.destination_dir / "subfolder" / "file3.txt").exists()) + + def test_copy_nonexistent_file(self): + # Test copying a nonexistent file (should not raise error, just warn) + source_file = self.source_dir / "nonexistent.txt" + try: + PartialCopy.partial_copy(source_file, self.source_dir, self.destination_dir) + # Should not raise error, just print warning + except Exception as e: + self.fail(f"partial_copy raised unexpected exception: {e}") + + def test_copy_nonexistent_glob(self): + # Test copying with a glob pattern that matches no files + glob_pattern = str(self.source_dir / "*_suffix.md") + glob_pattern = Cli.prepare_list_of_file(glob_pattern, self.destination_dir) + try: + PartialCopy.partial_copy(glob_pattern, self.source_dir, self.destination_dir) + # Should not raise error for empty glob results + except Exception as e: + self.fail(f"partial_copy raised unexpected exception: {e}") + + def test_copy_to_nonexistent_destination(self): + # Test copying to a nonexistent destination (should create the destination folder) + new_destination = self.destination_dir / "new_folder" + PartialCopy.partial_copy(self.source_dir / "file1.txt", self.source_dir, new_destination) + + # Check if the file was copied and the destination folder was created + self.assertTrue(new_destination.exists()) + self.assertTrue((new_destination / "file1.txt").exists()) + + def test_copy_path_object(self): + # Test copying using Path objects + source_file = self.source_dir / "file1.txt" + PartialCopy.partial_copy(source_file, self.source_dir, self.destination_dir) + + # Check if the file was copied + self.assertTrue((self.destination_dir / "file1.txt").exists()) + + def test_copy_text_file(self): + # Test copying a text file + PartialCopy.partial_copy(str(self.source_dir / "file1.txt"), self.source_dir, self.destination_dir) + + # Check if the file was copied + self.assertTrue((self.destination_dir / "file1.txt").exists()) + self.assertEqual((self.destination_dir / "file1.txt").read_text(), "Hello, file1!") + + def test_copy_markdown_file(self): + # Test copying a Markdown file + PartialCopy.partial_copy(str(self.source_dir / "file2.md"), self.source_dir, self.destination_dir) + + # Check if the file was copied with frontmatter + self.assertTrue((self.destination_dir / "file2.md").exists()) + content = (self.destination_dir / "file2.md").read_text() + self.assertIn("# Header\nSome content", content) + + def test_copy_directory_structure_with_header(self): + # Test copying a file while preserving directory structure + PartialCopy.partial_copy( + str(self.source_dir / "subfolder" / "file3.md"), + self.source_dir, + self.destination_dir + ) + + # Check if the file was copied with the directory structure + self.assertTrue((self.destination_dir / "subfolder" / "file3.md").exists()) + content = (self.destination_dir / "subfolder" / "file3.md").read_text() + self.assertIn("# Another Header\nMore content", content) + + def test_copy_referenced_images(self): + # Create a Markdown file with image references + md_content = "# Header\n![Image 1](images/image1.png)\n![Image 2](images/image2.jpg)" + (self.source_dir / "file1.md").write_text(md_content) + + # Create referenced images + (self.source_dir / "images").mkdir() + (self.source_dir / "images" / "image1.png").write_text("Fake PNG content") + (self.source_dir / "images" / "image2.jpg").write_text("Fake JPG content") + + # Copy files + PartialCopy.partial_copy(str(self.source_dir / "file1.md"), self.source_dir, self.destination_dir) + + # Check if the Markdown file was copied + self.assertTrue((self.destination_dir / "file1.md").exists()) + + # Check if referenced images were copied + self.assertTrue((self.destination_dir / "images" / "image1.png").exists()) + self.assertTrue((self.destination_dir / "images" / "image2.jpg").exists()) + + def test_copy_referenced_images_with_annotation(self): + # Create a Markdown file with image references + md_content = "# Header\n![Image 1](images/image1.png \"annotation\")\n![Image 2](images/image2.jpg)" + (self.source_dir / "file1.md").write_text(md_content) + + # Create referenced images + (self.source_dir / "images").mkdir() + (self.source_dir / "images" / "image1.png").write_text("Fake PNG content") + (self.source_dir / "images" / "image2.jpg").write_text("Fake JPG content") + + # Copy files + PartialCopy.partial_copy(str(self.source_dir / "file1.md"), self.source_dir, self.destination_dir) + + # Check if the Markdown file was copied + self.assertTrue((self.destination_dir / "file1.md").exists()) + + # Check if referenced images were copied + self.assertTrue((self.destination_dir / "images" / "image1.png").exists()) + self.assertTrue((self.destination_dir / "images" / "image2.jpg").exists()) + + def test_copy_with_string_paths(self): + # Test copying with string paths instead of Path objects + source_path = str(self.source_dir / "file1.txt") + root_path = str(self.source_dir) + dest_path = str(self.destination_dir) + + PartialCopy.partial_copy(source_path, root_path, dest_path) + + # Check if the file was copied + self.assertTrue((self.destination_dir / "file1.txt").exists()) + + def test_copy_empty_list(self): + # Test copying an empty list of files + try: + PartialCopy.partial_copy([], self.source_dir, self.destination_dir) + # Should not raise error for empty list + except Exception as e: + self.fail(f"partial_copy raised unexpected exception for empty list: {e}") + + def test_has_frontmatter_method(self): + content = (self.source_dir / "file4.md").read_text() + post = frontmatter.loads(content) + self.assertTrue(PartialCopy._has_frontmatter(post)) + + # Test without metadata + content = (self.source_dir / "file2.md").read_text() + post = frontmatter.loads(content) + self.assertFalse(PartialCopy._has_frontmatter(post)) + + +if __name__ == '__main__': + import unittest + unittest.main()