LCORE-3051: In the rag-content images, only install what is needed

.tekton/integration-tests/pipeline/rag-content-0-8-integration-test.yaml

@@ -292,7 +292,7 @@ spec:
                 -e STORAGE_DRIVER=vfs \\
                 ${PODMAN_ENV[*]} \\
                 registry.access.redhat.com/ubi9/ubi:latest \\
-                bash -c "dnf -y install podman fuse-overlayfs python3 curl-minimal jq && $TEST_CMD"
+                bash -c "dnf -y install podman fuse-overlayfs python3 curl-minimal jq diffutils && $TEST_CMD"
               SCRIPTEOF
               chmod +x scripts/test.sh
 
@@ -466,8 +466,8 @@ spec:
                 echo "CDI devices:"
                 nvidia-ctk cdi list || true
               fi
-              # Install any missing deps (best effort)
-              sudo dnf -y install python3 curl-minimal jq 2>/dev/null || true
+              # Install required test deps
+              sudo dnf -y install python3 jq diffutils
               cd $HOME/rag-content
               SCRIPTEOF
               # Inject env vars

Containerfile

@@ -8,10 +8,8 @@ ARG TARGETARCH
 USER root
 
 # Install Python and build tools.
-RUN ${BUILDER_DNF_COMMAND} -y module enable ruby:3.3 && \
-    ${BUILDER_DNF_COMMAND} install -y --nodocs --setopt=keepcache=0 --setopt=tsflags=nodocs \
-    python3.12 python3.12-devel python3.12-pip \
-    rubygems rubygem-bundler && \
+RUN ${BUILDER_DNF_COMMAND} install -y --nodocs --setopt=keepcache=0 --setopt=tsflags=nodocs \
+    python3.12 python3.12-devel python3.12-pip && \
     ${BUILDER_DNF_COMMAND} clean all
 
 
@@ -20,16 +18,6 @@ RUN pip3.12 install "uv>=0.7.20"
 
 WORKDIR /rag-content
 
-COPY Makefile pyproject.toml uv.lock README.md Gemfile Gemfile.lock .konflux/requirements.hashes.wheel.txt .konflux/requirements.hashes.wheel.pypi.txt .konflux/requirements.hashes.source.txt .konflux/requirements-build.txt ./
-COPY src ./src
-COPY tests ./tests
-COPY scripts ./scripts
-COPY embeddings_model ./embeddings_model
-COPY LICENSE /licenses/LICENSE
-
-# Install Ruby Gems
-RUN BUNDLE_PATH__SYSTEM=true bundle install
-
 # Configure UV environment variables for optimal performance
 # Pytorch backend - cpu. `uv` contains convenient way to specify the backend.
 # MATURIN_NO_INSTALL_RUST=1 : Disable installation of Rust dependencies by Maturin.
@@ -38,10 +26,9 @@ ENV UV_COMPILE_BYTECODE=0 \
     UV_PYTHON_DOWNLOADS=0 \
     MATURIN_NO_INSTALL_RUST=1
 
-WORKDIR /rag-content
-
 COPY pyproject.toml uv.lock README.md .konflux/requirements.hashes.wheel.txt .konflux/requirements.hashes.wheel.pypi.txt .konflux/requirements.hashes.source.txt .konflux/requirements-build.txt ./
 COPY src ./src
+COPY LICENSE /licenses/LICENSE
 
 RUN if [ -f /cachi2/cachi2.env ]; then \
     . /cachi2/cachi2.env && \
@@ -65,7 +52,7 @@ USER root
 
 RUN ${RUNTIME_DNF_COMMAND} -y module enable ruby:3.3 && \
     ${RUNTIME_DNF_COMMAND} install -y --nodocs --setopt=keepcache=0 --setopt=tsflags=nodocs \
-    python3.12 python3.12-pip \
+    python3.12 \
     libpq libxml2 libxslt libjpeg-turbo libtiff freetype libwebp \
     rubygems rubygem-bundler \
     skopeo && \
@@ -77,16 +64,16 @@ WORKDIR /rag-content
 # Copy the built venv from the builder stage.
 COPY --from=builder --chown=1000:1000 /rag-content/.venv .venv
 
-COPY Makefile pyproject.toml uv.lock README.md Gemfile Gemfile.lock ./
+# Install asciidoctor gem (for .adoc preprocessing)
+COPY Gemfile Gemfile.lock ./
+RUN if [ -f /cachi2/cachi2.env ]; then . /cachi2/cachi2.env; fi && \
+    BUNDLE_PATH__SYSTEM=true bundle install
+
 COPY src ./src
-COPY tests ./tests
-COPY scripts ./scripts
+COPY scripts/generate_embeddings.py scripts/download_embeddings_model.py ./scripts/
 COPY embeddings_model ./embeddings_model
 COPY LICENSE /licenses/LICENSE
 
-# Install Ruby Gems
-RUN BUNDLE_PATH__SYSTEM=true bundle install
-
 ENV PATH="/rag-content/.venv/bin:$PATH" \
     PYTHONPATH="/rag-content/src"
 

Containerfile-cuda

@@ -36,22 +36,17 @@ RUN pip3.12 install "uv>=0.7.20"
 
 WORKDIR /rag-content
 
-COPY Makefile pyproject.toml uv.lock README.md Gemfile Gemfile.lock .konflux/requirements.hashes.wheel.cuda.txt .konflux/requirements.hashes.source.cuda.txt .konflux/requirements.hashes.wheel.pypi.cuda.txt .konflux/requirements-build.cuda.txt ./
-COPY src ./src
-COPY tests ./tests
-COPY scripts ./scripts
-COPY embeddings_model ./embeddings_model
-COPY LICENSE /licenses/LICENSE
-
-# Install Ruby Gems
-RUN BUNDLE_PATH__SYSTEM=true bundle install
-
 # Configure UV environment variables for optimal performance
 ENV UV_COMPILE_BYTECODE=0 \
     UV_LINK_MODE=copy \
     UV_PYTHON_DOWNLOADS=0 \
     MATURIN_NO_INSTALL_RUST=1
 
+COPY pyproject.toml uv.lock README.md .konflux/requirements.hashes.wheel.cuda.txt .konflux/requirements.hashes.source.cuda.txt .konflux/requirements.hashes.wheel.pypi.cuda.txt .konflux/requirements-build.cuda.txt ./
+COPY src ./src
+COPY scripts/remove_pytorch_cpu_pyproject.py ./scripts/
+COPY LICENSE /licenses/LICENSE
+
 # If Cachi2 is present (hermetic CUDA build), install from prefetched CUDA requirement files.
 # faiss-cpu is direct wheel URL only (no sdist); use the arch-specific PyPI wheel file generated by scripts/konflux_requirements.sh cuda.
 # Otherwise install by removing pytorch-cpu index and re-locking so torch comes from PyPI  (CUDA).
@@ -79,7 +74,7 @@ USER root
 
 RUN ${RUNTIME_DNF_COMMAND} -y module enable ruby:3.3 && \
     ${RUNTIME_DNF_COMMAND} install -y --nodocs --setopt=keepcache=0 --setopt=tsflags=nodocs \
-    python3.12 python3.12-pip \
+    python3.12 \
     skopeo \
     libpq libxml2 libxslt libjpeg-turbo libtiff freetype libwebp \
     rubygems rubygem-bundler && \
@@ -99,16 +94,16 @@ WORKDIR /rag-content
 # Copy the built venv from the builder stage.
 COPY --from=builder --chown=1000:1000 /rag-content/.venv .venv
 
-COPY Makefile pyproject.toml uv.lock README.md Gemfile Gemfile.lock ./
+# Install asciidoctor gem (for .adoc preprocessing)
+COPY Gemfile Gemfile.lock ./
+RUN if [ -f /cachi2/cachi2.env ]; then . /cachi2/cachi2.env; fi && \
+    BUNDLE_PATH__SYSTEM=true bundle install
+
 COPY src ./src
-COPY tests ./tests
-COPY scripts ./scripts
+COPY scripts/generate_embeddings.py scripts/download_embeddings_model.py ./scripts/
 COPY embeddings_model ./embeddings_model
 COPY LICENSE /licenses/LICENSE
 
-# Install Ruby Gems
-RUN BUNDLE_PATH__SYSTEM=true bundle install
-
 ENV PATH="/rag-content/.venv/bin:$PATH" \
     PYTHONPATH="/rag-content/src"
 

src/lightspeed_rag_content/asciidoc/ruby_asciidoc/asciidoc_text_converter.rb

@@ -46,7 +46,7 @@ def convert_document(node)
       result = []
       result << "# " + decode(node.doctitle) if node.header?
       result << node.blocks.map { |child| convert(child) }.join("\n\n")
-      result.join("\n\n")
+      result.join("\n\n") + "\n"
     end
 
     def convert_section(node)

tests/asciidoc/fixture.adoc

@@ -0,0 +1,36 @@
+= Getting Started with Lightspeed
+
+== Prerequisites
+
+Before installing, ensure you have cluster administrator access
+to your OpenShift environment.
+
+The following tools must be available:
+
+* `oc` command-line client
+* `podman` for building container images
+* Access to a container registry
+
+== Installation
+
+Enable the feature by running the following command:
+
+[source,shell]
+----
+oc apply -f config.yaml
+----
+
+Verify the installation succeeded:
+
+. Check that the pod is running.
+. Confirm the service endpoint responds.
+. Review the logs for errors.
+
+NOTE: The command requires admin privileges.
+
+== Configuration
+
+The main configuration file is `config.yaml`.
+
+proxy_url:: The URL of the upstream proxy.
+timeout:: Connection timeout in seconds.

tests/asciidoc/fixture.md

@@ -0,0 +1,41 @@
+# Getting Started with Lightspeed
+
+# Prerequisites
+
+Before installing, ensure you have cluster administrator access
+to your OpenShift environment.
+
+The following tools must be available:
+
+* oc command-line client
+* podman for building container images
+* Access to a container registry
+
+# Installation
+
+Enable the feature by running the following command:
+
+
+```shell
+oc apply -f config.yaml
+```
+
+
+Verify the installation succeeded:
+
+1. Check that the pod is running.
+2. Confirm the service endpoint responds.
+3. Review the logs for errors.
+
+
+[NOTE]
+----
+The command requires admin privileges.
+----
+
+# Configuration
+
+The main configuration file is config.yaml.
+
+proxy_url:: The URL of the upstream proxy.
+timeout:: Connection timeout in seconds.

tests/asciidoc/test_integration.py

@@ -0,0 +1,86 @@
+# Copyright 2025 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""End-to-end integration test: AsciiDoc conversion preserves content.
+
+Drives the real AsciidoctorConverter (requires ``asciidoctor`` to be installed)
+and verifies that the converted output matches the hand-written Markdown
+reference.  This validates the .adoc -> .md -> generate_embeddings.py path
+that users follow inside the tool container.
+"""
+
+import shutil
+from pathlib import Path
+
+import pytest
+
+from lightspeed_rag_content.asciidoc import AsciidoctorConverter
+
+FIXTURE_DIR = Path(__file__).parent
+FIXTURE_ADOC = FIXTURE_DIR / "fixture.adoc"
+FIXTURE_MD = FIXTURE_DIR / "fixture.md"
+
+requires_asciidoctor = pytest.mark.skipif(
+    not shutil.which("ruby") or not shutil.which("asciidoctor"),
+    reason="ruby and/or asciidoctor not installed (run 'bundle install' first)",
+)
+
+CONTENT_MARKERS = [
+    "Getting Started with Lightspeed",
+    "Prerequisites",
+    "cluster administrator access",
+    "OpenShift environment",
+    "oc",
+    "podman",
+    "container registry",
+    "Installation",
+    "oc apply -f config.yaml",
+    "pod is running",
+    "service endpoint responds",
+    "Review the logs for errors",
+    "admin privileges",
+    "Configuration",
+    "config.yaml",
+    "proxy_url",
+    "upstream proxy",
+    "timeout",
+    "Connection timeout in seconds",
+]
+
+
+@requires_asciidoctor
+class TestAsciidocIntegration:
+    def test_converted_output_matches_reference(self, tmp_path):
+        """The .adoc -> text conversion produces the same content as the .md reference."""
+        output_file = tmp_path / "converted.md"
+
+        converter = AsciidoctorConverter()
+        converter.convert(FIXTURE_ADOC, output_file)
+
+        converted = output_file.read_text(encoding="utf-8")
+        reference = FIXTURE_MD.read_text(encoding="utf-8")
+
+        assert converted.strip() == reference.strip()
+
+    def test_converted_output_preserves_all_content(self, tmp_path):
+        """Every key phrase from the source .adoc survives the conversion."""
+        output_file = tmp_path / "converted.md"
+
+        converter = AsciidoctorConverter()
+        converter.convert(FIXTURE_ADOC, output_file)
+
+        converted = output_file.read_text(encoding="utf-8")
+
+        for marker in CONTENT_MARKERS:
+            assert marker in converted, f"expected {marker!r} in converted text"