Merge branch 'langgenius:main' into main

api/Dockerfile

@@ -55,7 +55,12 @@ RUN apt-get update \
     && echo "deb http://deb.debian.org/debian testing main" > /etc/apt/sources.list \
     && apt-get update \
     # For Security
-    && apt-get install -y --no-install-recommends zlib1g=1:1.3.dfsg+really1.3.1-1 expat=2.6.3-1 libldap-2.5-0=2.5.18+dfsg-3+b1 perl=5.40.0-6 libsqlite3-0=3.46.1-1 \
+    && apt-get install -y --no-install-recommends expat=2.6.3-2 libldap-2.5-0=2.5.18+dfsg-3+b1 perl=5.40.0-6 libsqlite3-0=3.46.1-1 \
+    && if [ "$(dpkg --print-architecture)" = "amd64" ]; then \
+        apt-get install -y --no-install-recommends zlib1g=1:1.3.dfsg+really1.3.1-1+b1; \
+    else \
+        apt-get install -y --no-install-recommends zlib1g=1:1.3.dfsg+really1.3.1-1; \
+    fi \
     # install a chinese font to support the use of tools like matplotlib
     && apt-get install -y fonts-noto-cjk \
     && apt-get autoremove -y \

api/controllers/console/version.py

@@ -3,6 +3,7 @@
 
 import requests
 from flask_restful import Resource, reqparse
+from packaging import version
 
 from configs import dify_config
 
@@ -47,43 +48,15 @@ def get(self):
 
 
 def _has_new_version(*, latest_version: str, current_version: str) -> bool:
-    def parse_version(version: str) -> tuple:
-        # Split version into parts and pre-release suffix if any
-        parts = version.split("-")
-        version_parts = parts[0].split(".")
-        pre_release = parts[1] if len(parts) > 1 else None
-
-        # Validate version format
-        if len(version_parts) != 3:
-            raise ValueError(f"Invalid version format: {version}")
-
-        try:
-            # Convert version parts to integers
-            major, minor, patch = map(int, version_parts)
-            return (major, minor, patch, pre_release)
-        except ValueError:
-            raise ValueError(f"Invalid version format: {version}")
-
-    latest = parse_version(latest_version)
-    current = parse_version(current_version)
-
-    # Compare major, minor, and patch versions
-    for latest_part, current_part in zip(latest[:3], current[:3]):
-        if latest_part > current_part:
-            return True
-        elif latest_part < current_part:
-            return False
-
-    # If versions are equal, check pre-release suffixes
-    if latest[3] is None and current[3] is not None:
-        return True
-    elif latest[3] is not None and current[3] is None:
+    try:
+        latest = version.parse(latest_version)
+        current = version.parse(current_version)
+
+        # Compare versions
+        return latest > current
+    except version.InvalidVersion:
+        logging.warning(f"Invalid version format: latest={latest_version}, current={current_version}")
         return False
-    elif latest[3] is not None and current[3] is not None:
-        # Simple string comparison for pre-release versions
-        return latest[3] > current[3]
-
-    return False
 
 
 api.add_resource(VersionApi, "/version")

api/controllers/service_api/dataset/document.py

@@ -331,10 +331,26 @@ def get(self, tenant_id, dataset_id, batch):
         return data
 
 
-api.add_resource(DocumentAddByTextApi, "/datasets/<uuid:dataset_id>/document/create_by_text")
-api.add_resource(DocumentAddByFileApi, "/datasets/<uuid:dataset_id>/document/create_by_file")
-api.add_resource(DocumentUpdateByTextApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update_by_text")
-api.add_resource(DocumentUpdateByFileApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update_by_file")
+api.add_resource(
+    DocumentAddByTextApi,
+    "/datasets/<uuid:dataset_id>/document/create_by_text",
+    "/datasets/<uuid:dataset_id>/document/create-by-text",
+)
+api.add_resource(
+    DocumentAddByFileApi,
+    "/datasets/<uuid:dataset_id>/document/create_by_file",
+    "/datasets/<uuid:dataset_id>/document/create-by-file",
+)
+api.add_resource(
+    DocumentUpdateByTextApi,
+    "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update_by_text",
+    "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update-by-text",
+)
+api.add_resource(
+    DocumentUpdateByFileApi,
+    "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update_by_file",
+    "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update-by-file",
+)
 api.add_resource(DocumentDeleteApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>")
 api.add_resource(DocumentListApi, "/datasets/<uuid:dataset_id>/documents")
 api.add_resource(DocumentIndexingStatusApi, "/datasets/<uuid:dataset_id>/documents/<string:batch>/indexing-status")

api/controllers/service_api/dataset/hit_testing.py

@@ -14,4 +14,4 @@ def post(self, tenant_id, dataset_id):
         return self.perform_hit_testing(dataset, args)
 
 
-api.add_resource(HitTestingApi, "/datasets/<uuid:dataset_id>/hit-testing")
+api.add_resource(HitTestingApi, "/datasets/<uuid:dataset_id>/hit-testing", "/datasets/<uuid:dataset_id>/retrieve")

api/core/rag/rerank/rerank_model.py

@@ -27,18 +27,17 @@ def run(
         :return:
         """
         docs = []
-        doc_id = []
+        doc_id = set()
         unique_documents = []
-        dify_documents = [item for item in documents if item.provider == "dify"]
-        external_documents = [item for item in documents if item.provider == "external"]
-        for document in dify_documents:
-            if document.metadata["doc_id"] not in doc_id:
-                doc_id.append(document.metadata["doc_id"])
+        for document in documents:
+            if document.provider == "dify" and document.metadata["doc_id"] not in doc_id:
+                doc_id.add(document.metadata["doc_id"])
                 docs.append(document.page_content)
                 unique_documents.append(document)
-        for document in external_documents:
-            docs.append(document.page_content)
-            unique_documents.append(document)
+            elif document.provider == "external":
+                if document not in unique_documents:
+                    docs.append(document.page_content)
+                    unique_documents.append(document)
 
         documents = unique_documents
 

api/models/model.py

@@ -396,7 +396,7 @@ def to_dict(self) -> dict:
             "file_upload": self.file_upload_dict,
         }
 
-    def from_model_config_dict(self, model_config: dict):
+    def from_model_config_dict(self, model_config: Mapping[str, Any]):
         self.opening_statement = model_config.get("opening_statement")
         self.suggested_questions = (
             json.dumps(model_config["suggested_questions"]) if model_config.get("suggested_questions") else None

api/services/app_dsl_service/__init__.py

@@ -0,0 +1,3 @@
+from .service import AppDslService
+
+__all__ = ["AppDslService"]

api/services/app_dsl_service/exc.py

@@ -0,0 +1,34 @@
+class DSLVersionNotSupportedError(ValueError):
+    """Raised when the imported DSL version is not supported by the current Dify version."""
+
+
+class InvalidYAMLFormatError(ValueError):
+    """Raised when the provided YAML format is invalid."""
+
+
+class MissingAppDataError(ValueError):
+    """Raised when the app data is missing in the provided DSL."""
+
+
+class InvalidAppModeError(ValueError):
+    """Raised when the app mode is invalid."""
+
+
+class MissingWorkflowDataError(ValueError):
+    """Raised when the workflow data is missing in the provided DSL."""
+
+
+class MissingModelConfigError(ValueError):
+    """Raised when the model config data is missing in the provided DSL."""
+
+
+class FileSizeLimitExceededError(ValueError):
+    """Raised when the file size exceeds the allowed limit."""
+
+
+class EmptyContentError(ValueError):
+    """Raised when the content fetched from the URL is empty."""
+
+
+class ContentDecodingError(ValueError):
+    """Raised when there is an error decoding the content."""