diff --git a/development/docs/build_block_docs.py b/development/docs/build_block_docs.py index 2d7205c911..602da94d26 100644 --- a/development/docs/build_block_docs.py +++ b/development/docs/build_block_docs.py @@ -5,6 +5,7 @@ from typing import Dict, List, Set, Tuple, Type import inspect + from inference.core.utils.file_system import dump_text_lines, read_text_file from inference.core.workflows.execution_engine.entities.base import OutputDefinition from inference.core.workflows.execution_engine.entities.types import STEP_AS_SELECTED_ELEMENT @@ -22,6 +23,16 @@ ) from inference.core.workflows.prototypes.block import WorkflowBlock + +from jinja2 import Environment, FileSystemLoader + +template_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "templates") +jinja_env = Environment(loader=FileSystemLoader(template_dir)) + +def render_template(template_name, **kwargs): + template = jinja_env.get_template(template_name) + return template.render(**kwargs) + DOCS_ROOT_DIR = os.path.abspath( os.path.join( os.path.dirname(__file__), @@ -32,7 +43,7 @@ ) BLOCKS_DIR = os.path.join(DOCS_ROOT_DIR, "workflows", "blocks") -BLOCKS_DOCUMENTATION_TEMPLATE = os.path.join(DOCS_ROOT_DIR, "workflows", "blocks_gallery_template.md") + BLOCK_DOCUMENTATION_FILE = os.path.join(DOCS_ROOT_DIR, "workflows", "blocks", "index.md") KINDS_DIR = os.path.join(DOCS_ROOT_DIR, "workflows", "kinds") @@ -41,7 +52,6 @@ BLOCK_DOCUMENTATION_DIRECTORY = os.path.join(DOCS_ROOT_DIR, "workflows", "blocks") KINDS_DOCUMENTATION_DIRECTORY = os.path.join(DOCS_ROOT_DIR, "workflows", "kinds") -AUTOGENERATED_BLOCKS_LIST_TOKEN = "" AUTOGENERATED_KINDS_LIST_TOKEN = "" USER_CONFIGURATION_HEADER = [ @@ -187,113 +197,103 @@ INLINE_UQL_PARAMETER_PATTERN = re.compile(r"({{\s*\$parameters\.(\w+)\s*}})") +BLOCK_SECTIONS = [ + { + "title": "Models", + "id": "model", + "colorScheme": "purboflow" + }, + { + "title": "Visualizations", + "id": "visualization", + "colorScheme": "blue" + }, + { + "title": "Logic and Branching", + "id": "flow_control", + "colorScheme": "yellow" + }, + { + "title": "Data Storage", + "id": "data_storage", + "colorScheme": "pink" + }, + { + "title": "Notifications", + "id": "notifications", + "colorScheme": "salmon" + }, + { + "title": "Transformations", + "id": "transformation", + "colorScheme": "green" + }, + { + "title": "Classical Computer Vision", + "id": "classical_cv", + "colorScheme": "cyan" + }, + { + "title": "Video", + "id": "video", + "colorScheme": "indigo" + }, + { + "title": "Advanced", + "id": "advanced", + "colorScheme": "orange" + } + ] def main() -> None: - os.makedirs(BLOCK_DOCUMENTATION_DIRECTORY, exist_ok=True) - os.makedirs(KINDS_DOCUMENTATION_DIRECTORY, exist_ok=True) - lines = read_text_file( - path=BLOCKS_DOCUMENTATION_TEMPLATE, - split_lines=True, - ) - start_index, end_index = get_auto_generation_markers( - documentation_lines=lines, - token=AUTOGENERATED_BLOCKS_LIST_TOKEN, - ) - block_card_lines = [] blocks_description = describe_available_blocks(dynamic_blocks=[]) + write_kinds_docs(blocks_description) + write_blocks_docs(blocks_description) + + +def write_blocks_docs(blocks_description): + + # create blocks directory if it doesn't exist + os.makedirs(BLOCK_DOCUMENTATION_DIRECTORY, exist_ok=True) + + # get block assgined to families + block_families = get_block_families(blocks_description) + + # write blocks index file + write_blocks_index_file(block_families) + + # write blocks summary file + write_blocks_summary_md(block_families) + + # write individual block pages + write_individual_block_pages(block_families, blocks_description) + + +def write_individual_block_pages(block_families, blocks_description): block_type2manifest_type_identifier = { block.block_class: block.human_friendly_block_name for block in blocks_description.blocks } blocks_connections = discover_blocks_connections( blocks_description=blocks_description - ) - generated_kinds_index_lines = [] - for declared_kind in blocks_description.declared_kinds: - description = ( - declared_kind.description - if declared_kind.description is not None - else "Not available." - ) - details = ( - declared_kind.docs if declared_kind.docs is not None else "Not available." - ) - warning = "" - if declared_kind.internal_data_type != declared_kind.serialised_data_type: - warning = DATA_REPRESENTATION_WARNING - kind_page = KIND_PAGE_TEMPLATE.format( - kind_name=declared_kind.name, - description=description, - details=details, - data_representation_warning=warning, - serialised_data_type=declared_kind.serialised_data_type, - internal_data_type=declared_kind.internal_data_type, - ) - relative_link = ( - f"../kinds/{slugify_kind_name(kind_name=declared_kind.name)}.md" - ) - generated_kinds_index_lines.append( - f"* [`{declared_kind.name}`]({relative_link}): {description}\n" - ) - kind_file_path = build_kind_page_path(kind_name=declared_kind.name) - with open(kind_file_path, "w") as documentation_file: - documentation_file.write(kind_page) - - generated_kinds_index_lines = sorted(generated_kinds_index_lines) - write_kinds_summary_md(generated_kinds_index_lines) + ) - kinds_index_lines = read_text_file( - path=KINDS_DOCUMENTATION_TEMPLATE, - split_lines=True, - ) - kinds_start_index, kinds_end_index = get_auto_generation_markers( - documentation_lines=kinds_index_lines, - token=AUTOGENERATED_KINDS_LIST_TOKEN, - ) - kinds_index_lines = ( - kinds_index_lines[: kinds_start_index + 1] - + generated_kinds_index_lines - + kinds_index_lines[kinds_end_index:] - ) - dump_text_lines( - path=KINDS_DOCUMENTATION_FILE, - content=kinds_index_lines, - allow_override=True, - lines_connector="", - ) - block_families = defaultdict(list) - for block in blocks_description.blocks: - block_families[block.human_friendly_block_name].append(block) for family_name, family_members in block_families.items(): - block_families[family_name] = sorted( - family_members, - key=lambda block: int(block.block_schema.get("version", "v0")[1:]), - reverse=True, - ) - for family_name, family_members in block_families.items(): - block_types_in_family = set() - block_licenses_in_family = set() + documentation_file_name = slugify_block_name(family_name) + ".md" documentation_file_path = os.path.join( BLOCK_DOCUMENTATION_DIRECTORY, documentation_file_name ) - short_descriptions = [] + versions_content = [] for block in family_members: - - block_type = block.block_schema.get("block_type", "").upper() - block_class_name = block.fully_qualified_block_class_name block_source_link = get_source_link_for_block_class(block.block_class) - block_types_in_family.add(block_type) - block_license = block.block_schema.get("license", "").upper() - block_licenses_in_family.add(block_license) example_definition = generate_example_step_definition(block=block) parsed_manifest = parse_block_manifest(manifest_type=block.manifest_class) long_description = block.block_schema.get("long_description", "Description not available") - short_description = block.block_schema.get("short_description", "Description not available") - short_descriptions.append(short_description) + template = BLOCK_VERSION_TEMPLATE_SINGLE_VERSION if len(family_members) == 1 else BLOCK_VERSION_TEMPLATE_MULTIPLE_VERSIONS @@ -330,25 +330,102 @@ def main() -> None: ) with open(documentation_file_path, "w") as documentation_file: documentation_file.write(family_document_content) - block_card_line = BLOCK_CARD_TEMPLATE.format( - data_url=slugify_block_name(family_name), - data_name=family_name, - data_desc=short_descriptions[-1], - data_labels=", ".join(list(block_types_in_family) + list(block_licenses_in_family)), - data_authors="dummy", - ) - block_card_lines.append(block_card_line) - block_card_lines = sorted(block_card_lines) - lines = lines[: start_index + 1] + block_card_lines + lines[end_index:] - dump_text_lines( - path=BLOCK_DOCUMENTATION_FILE, - content=lines, - allow_override=True, - lines_connector="", - ) - write_blocks_summary_md(block_families) + + +def get_block_families_by_section(block_families): + # Group families by block_type + blocks_by_section = defaultdict(list) + for family_name, members in block_families.items(): + if not members: + section = "custom" + else: + block_name = members[0].block_schema.get("name", "Missing Name") + ui_manifest = members[0].block_schema.get("ui_manifest", {}) + section = ui_manifest.get("section", "custom") + if not section: + section = "custom" + blocks_by_section[section].append(family_name) + + return blocks_by_section + + + +def write_blocks_summary_md(block_families): + """ + Creates SUMMARY.md for mkdocs-literate-nav. + """ + + block_families_by_section = get_block_families_by_section(block_families) + + lines = [] + + # For each block type, create a top-level bullet, then sub-bullets for families + for block_section in BLOCK_SECTIONS: + section_title = block_section['title'] + section_id = block_section['id'] + + lines.append(f"* {section_title}") + for family_name in sorted(block_families_by_section[section_id], key=lambda x: block_families[x][0].block_schema.get("ui_manifest", {}).get("blockPriority", 99)): + # Suppose you had a function slugify_block_name: + slug = slugify_block_name(family_name) + # Link to foo.md (or bar.md, etc.) + lines.append(f" * [{family_name}]({slug}.md)") + + summary_path = os.path.join(BLOCKS_DIR, "SUMMARY.md") + with open(summary_path, "w", encoding="utf-8") as f: + f.write("\n".join(lines) + "\n") + + +def write_blocks_index_file(block_families): + + block_families_by_section = get_block_families_by_section(block_families) + blocks_by_section = {} + + for block_section in BLOCK_SECTIONS: + section_title = block_section['title'] + section_id = block_section['id'] + + blocks_by_section[section_id] = [] + + + for family_name in sorted(block_families_by_section[section_id], key=lambda x: block_families[x][0].block_schema.get("ui_manifest", {}).get("blockPriority", 99)): + block_schema = block_families[family_name][0].block_schema + block_data = { + "name": family_name, + "url": slugify_block_name(family_name), + "description": block_schema.get("short_description", "Description not available"), + "license": block_schema.get("license", "").upper(), + "icon": block_schema.get("ui_manifest", {}).get("icon", "far fa-sparkles") + } + blocks_by_section[section_id].append(block_data) + + + + + output = render_template("blocks_index.md", blocks_by_section=blocks_by_section, block_sections=BLOCK_SECTIONS) + + + with open(BLOCK_DOCUMENTATION_FILE, "w", encoding="utf-8") as f: + f.write(output) + + +def get_block_families(blocks_description): + ''' + Get block families and sort them by version. + ''' + block_families = defaultdict(list) + for block in blocks_description.blocks: + block_families[block.human_friendly_block_name].append(block) + for family_name, family_members in block_families.items(): + block_families[family_name] = sorted( + family_members, + key=lambda block: int(block.block_schema.get("version", "v0")[1:]), + reverse=True, + ) + return block_families + def combined_content_from_versions(versions_content: List[str]) -> str: return "\n\n".join(versions_content) @@ -545,37 +622,69 @@ def write_kinds_summary_md(kinds): with open(summary_path, "w", encoding="utf-8") as f: f.write("".join(lines) + "\n") -def write_blocks_summary_md(block_families): - """ - Creates SUMMARY.md for mkdocs-literate-nav. - """ - # Group families by block_type - blocks_by_type = defaultdict(list) - for family_name, members in block_families.items(): - if not members: - block_type = "OTHER" - else: - block_type = members[0].block_schema.get("block_type", "OTHER") - if not block_type: - block_type = "OTHER" - blocks_by_type[block_type].append(family_name) - lines = [] - # For each block type, create a top-level bullet, then sub-bullets for families - for block_type in sorted(blocks_by_type.keys()): - type_title = to_title_case(block_type) - lines.append(f"* {type_title}") - for family_name in sorted(blocks_by_type[block_type]): - # Suppose you had a function slugify_block_name: - slug = slugify_block_name(family_name) - # Link to foo.md (or bar.md, etc.) - lines.append(f" * [{family_name}]({slug}.md)") +def write_kinds_docs(blocks_description): + os.makedirs(KINDS_DOCUMENTATION_DIRECTORY, exist_ok=True) + + generated_kinds_index_lines = [] + for declared_kind in blocks_description.declared_kinds: + + description = ( + declared_kind.description + if declared_kind.description is not None + else "Not available." + ) + details = ( + declared_kind.docs if declared_kind.docs is not None else "Not available." + ) + warning = "" + if declared_kind.internal_data_type != declared_kind.serialised_data_type: + warning = DATA_REPRESENTATION_WARNING + kind_page = KIND_PAGE_TEMPLATE.format( + kind_name=declared_kind.name, + description=description, + details=details, + data_representation_warning=warning, + serialised_data_type=declared_kind.serialised_data_type, + internal_data_type=declared_kind.internal_data_type, + ) + relative_link = ( + f"../kinds/{slugify_kind_name(kind_name=declared_kind.name)}.md" + ) + generated_kinds_index_lines.append( + f"* [`{declared_kind.name}`]({relative_link}): {description}\n" + ) + kind_file_path = build_kind_page_path(kind_name=declared_kind.name) + with open(kind_file_path, "w") as documentation_file: + documentation_file.write(kind_page) + + generated_kinds_index_lines = sorted(generated_kinds_index_lines) + write_kinds_summary_md(generated_kinds_index_lines) + + kinds_index_lines = read_text_file( + path=KINDS_DOCUMENTATION_TEMPLATE, + split_lines=True, + ) + kinds_start_index, kinds_end_index = get_auto_generation_markers( + documentation_lines=kinds_index_lines, + token=AUTOGENERATED_KINDS_LIST_TOKEN, + ) + kinds_index_lines = ( + kinds_index_lines[: kinds_start_index + 1] + + generated_kinds_index_lines + + kinds_index_lines[kinds_end_index:] + ) + dump_text_lines( + path=KINDS_DOCUMENTATION_FILE, + content=kinds_index_lines, + allow_override=True, + lines_connector="", + ) - summary_path = os.path.join(BLOCKS_DIR, "SUMMARY.md") - with open(summary_path, "w", encoding="utf-8") as f: - f.write("\n".join(lines) + "\n") +def write_blocks_gallery(): + pass if __name__ == "__main__": main() \ No newline at end of file diff --git a/development/docs/templates/blocks_index.md b/development/docs/templates/blocks_index.md new file mode 100644 index 0000000000..085f328805 --- /dev/null +++ b/development/docs/templates/blocks_index.md @@ -0,0 +1,79 @@ +--- +hide: + - toc +--- + + + + + + +
+
+
+

Workflow Blocks

+
+ + {% for section in block_sections %} +
+

{{ section.title | capitalize }}

+
+
+ {% for block in blocks_by_section[section.id] %} + + + + +
+ +
{{ block.name }}
+
{{ block.description }}
+ +
+ + + + + {% endfor %} +
+
+
+ {% endfor %} +
+ +
+ + \ No newline at end of file diff --git a/docs/api.md b/docs/api.md index 55f2a6331c..13be2f7525 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,3 +1,7 @@ +--- +hide: + - toc +--- When the Inference Server is running, it provides OpenAPI documentation at the `/docs` endpoint for use in development. Below is the OpenAPI specification for the Inference Server for the current release version. diff --git a/inference/core/workflows/core_steps/classical_cv/dominant_color/v1.py b/inference/core/workflows/core_steps/classical_cv/dominant_color/v1.py index 205670ab3b..af1bf6ed05 100644 --- a/inference/core/workflows/core_steps/classical_cv/dominant_color/v1.py +++ b/inference/core/workflows/core_steps/classical_cv/dominant_color/v1.py @@ -44,6 +44,12 @@ class DominantColorManifest(WorkflowBlockManifest): "long_description": LONG_DESCRIPTION, "license": "Apache-2.0", "block_type": "classical_computer_vision", + "ui_manifest": { + "section": "classical_cv", + "icon": "far fa-palette", + "blockPriority": 1, + "opencv": True, + }, } ) image: Selector(kind=[IMAGE_KIND]) = Field( diff --git a/inference/core/workflows/core_steps/classical_cv/image_blur/v1.py b/inference/core/workflows/core_steps/classical_cv/image_blur/v1.py index 1ab40828ac..ddceb0a376 100644 --- a/inference/core/workflows/core_steps/classical_cv/image_blur/v1.py +++ b/inference/core/workflows/core_steps/classical_cv/image_blur/v1.py @@ -40,6 +40,12 @@ class ImageBlurManifest(WorkflowBlockManifest): "long_description": LONG_DESCRIPTION, "license": "Apache-2.0", "block_type": "classical_computer_vision", + "ui_manifest": { + "section": "classical_cv", + "icon": "far fa-droplet", + "blockPriority": 5, + "opencv": True, + }, } ) diff --git a/inference/core/workflows/core_steps/models/foundation/ocr/v1.py b/inference/core/workflows/core_steps/models/foundation/ocr/v1.py index b560032e89..bb774f7394 100644 --- a/inference/core/workflows/core_steps/models/foundation/ocr/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/ocr/v1.py @@ -67,6 +67,13 @@ class BlockManifest(WorkflowBlockManifest): "long_description": LONG_DESCRIPTION, "license": "Apache-2.0", "block_type": "model", + "ui_manifest": { + "section": "model", + "icon": "far fa-text", + "blockPriority": 11, + "inDevelopment": True, + "inference": True, + }, } ) type: Literal["roboflow_core/ocr_model@v1", "OCRModel"] diff --git a/inference/core/workflows/core_steps/sinks/email_notification/v1.py b/inference/core/workflows/core_steps/sinks/email_notification/v1.py index 64da9c7cbf..5ad2b7d18f 100644 --- a/inference/core/workflows/core_steps/sinks/email_notification/v1.py +++ b/inference/core/workflows/core_steps/sinks/email_notification/v1.py @@ -172,7 +172,7 @@ class BlockManifest(WorkflowBlockManifest): "license": "Apache-2.0", "block_type": "sink", "ui_manifest": { - "section": "model", + "section": "notifications", "icon": "far fa-envelope", "blockPriority": 0, "popular": True, diff --git a/inference/core/workflows/core_steps/sinks/local_file/v1.py b/inference/core/workflows/core_steps/sinks/local_file/v1.py index 790a345a2e..c4dfba2150 100644 --- a/inference/core/workflows/core_steps/sinks/local_file/v1.py +++ b/inference/core/workflows/core_steps/sinks/local_file/v1.py @@ -74,7 +74,7 @@ class BlockManifest(WorkflowBlockManifest): "license": "Apache-2.0", "block_type": "sink", "ui_manifest": { - "section": "model", + "section": "data_storage", "icon": "fal fa-file", "blockPriority": 3, "popular": True, diff --git a/inference/core/workflows/core_steps/transformations/absolute_static_crop/v1.py b/inference/core/workflows/core_steps/transformations/absolute_static_crop/v1.py index 755f04d029..35fbb95b32 100644 --- a/inference/core/workflows/core_steps/transformations/absolute_static_crop/v1.py +++ b/inference/core/workflows/core_steps/transformations/absolute_static_crop/v1.py @@ -39,6 +39,11 @@ class BlockManifest(WorkflowBlockManifest): "long_description": LONG_DESCRIPTION, "license": "Apache-2.0", "block_type": "transformation", + "ui_manifest": { + "section": "transformation", + "icon": "far fa-crop-alt", + "blockPriority": 1, + }, } ) type: Literal["roboflow_core/absolute_static_crop@v1", "AbsoluteStaticCrop"] diff --git a/inference/core/workflows/core_steps/visualizations/dot/v1.py b/inference/core/workflows/core_steps/visualizations/dot/v1.py index 2155b176ac..9c15ec0a97 100644 --- a/inference/core/workflows/core_steps/visualizations/dot/v1.py +++ b/inference/core/workflows/core_steps/visualizations/dot/v1.py @@ -38,6 +38,12 @@ class DotManifest(ColorableVisualizationManifest): "long_description": LONG_DESCRIPTION, "license": "Apache-2.0", "block_type": "visualization", + "ui_manifest": { + "section": "classical_cv", + "icon": "far fa-palette", + "blockPriority": 1, + "opencv": True, + }, } )