From f9163633a00ca555dd4ff7a27403e2b93e325409 Mon Sep 17 00:00:00 2001 From: Hari John Kuriakose Date: Thu, 15 Aug 2024 17:01:48 +0530 Subject: [PATCH 1/6] * Add error logging for fetching user orgs * Improve logging and docs * Fix protobuf generated code version mismtach issue * Add settings/celery.py * Rename celery instance to celery * Refactor celery services config * Rename settings/dev.py to settings/platform.py * Fix env and django settings loading order in backend entrypoints * Remove django settings module update from utils * Remove unnecessary flowerconfig.py * Add env var to suppress evaluation server warnings * Add check for active venv in dev-env-cli.sh --- .gitignore | 3 +- backend/README.md | 2 +- backend/account/authentication_controller.py | 3 +- .../account/authentication_plugin_registry.py | 9 +- backend/backend/__init__.py | 3 - backend/backend/asgi.py | 10 +- .../backend/{celery_service.py => celery.py} | 22 ++-- backend/backend/flowerconfig.py | 15 --- backend/backend/settings/base.py | 44 +++----- backend/backend/settings/celery.py | 100 ++++++++++++++++++ .../backend/settings/{dev.py => platform.py} | 2 +- backend/backend/wsgi.py | 13 +-- backend/manage.py | 2 +- backend/sample.env | 4 +- backend/utils/celery/constants.py | 19 ++++ .../celery/task_registry.py} | 0 backend/utils/constants.py | 9 -- dev-env-cli.sh | 9 ++ unstract/flags/README.md | 62 +++++------ .../flags/src/unstract/flags/client/base.py | 4 + .../src/unstract/flags/client/evaluation.py | 19 ++-- unstract/flags/src/unstract/flags/sample.env | 5 +- 22 files changed, 230 insertions(+), 129 deletions(-) rename backend/backend/{celery_service.py => celery.py} (66%) delete mode 100644 backend/backend/flowerconfig.py create mode 100644 backend/backend/settings/celery.py rename backend/backend/settings/{dev.py => platform.py} (97%) create mode 100644 backend/utils/celery/constants.py rename backend/{backend/celery_task.py => utils/celery/task_registry.py} (100%) diff --git a/.gitignore b/.gitignore index 82fca7fce..3a88d7c94 100644 --- a/.gitignore +++ b/.gitignore @@ -639,7 +639,8 @@ tools/*/data_dir/ backend/backend/settings/* !backend/backend/settings/__init__.py !backend/backend/settings/base.py -!backend/backend/settings/dev.py +!backend/backend/settings/platform.py +!backend/backend/settings/celery.py !backend/backend/settings/test.py # Local Dependencies for docker testing diff --git a/backend/README.md b/backend/README.md index 40ebdf467..b84246acf 100644 --- a/backend/README.md +++ b/backend/README.md @@ -18,7 +18,7 @@ Contains the backend services for Unstract written with Django and DRF. - Copy `sample.env` into `.env` and update the necessary variables. For eg: ``` -DJANGO_SETTINGS_MODULE='backend.settings.dev' +DJANGO_SETTINGS_MODULE='backend.settings.platform' DB_HOST='localhost' DB_USER='unstract_dev' DB_PASSWORD='unstract_pass' diff --git a/backend/account/authentication_controller.py b/backend/account/authentication_controller.py index f8b41e921..f1160c14c 100644 --- a/backend/account/authentication_controller.py +++ b/backend/account/authentication_controller.py @@ -113,7 +113,8 @@ def user_organizations(self, request: Request) -> Any: try: organizations = self.auth_service.user_organizations(request) except Exception as ex: - # + Logger.error(f"Failed to get user orgs: {ex}") + self.user_logout(request) response = Response( diff --git a/backend/account/authentication_plugin_registry.py b/backend/account/authentication_plugin_registry.py index cd630fdaf..c3d7dbc04 100644 --- a/backend/account/authentication_plugin_registry.py +++ b/backend/account/authentication_plugin_registry.py @@ -43,14 +43,11 @@ def _load_plugins() -> dict[str, dict[str, Any]]: PluginConfig.AUTH_METADATA: module.metadata, } Logger.info( - "Loaded auth plugin: %s, is_active: %s", - module.metadata["name"], - module.metadata["is_active"], + "Loaded active authentication plugin: %s", module.metadata["name"] ) else: - Logger.warning( - "Metadata is not active for %s authentication module.", - auth_module_name, + Logger.info( + "Skipping inactive authentication plugin: %s", auth_module_name ) except ModuleNotFoundError as exception: Logger.error( diff --git a/backend/backend/__init__.py b/backend/backend/__init__.py index e921073a8..e69de29bb 100644 --- a/backend/backend/__init__.py +++ b/backend/backend/__init__.py @@ -1,3 +0,0 @@ -from .celery_service import app as celery_app - -__all__ = ["celery_app"] diff --git a/backend/backend/asgi.py b/backend/backend/asgi.py index 07774528a..b9687bca8 100644 --- a/backend/backend/asgi.py +++ b/backend/backend/asgi.py @@ -8,13 +8,17 @@ import os -from django.core.asgi import get_asgi_application -from dotenv import load_dotenv +from dotenv import find_dotenv, load_dotenv + +load_dotenv(find_dotenv() or "") load_dotenv() + os.environ.setdefault( "DJANGO_SETTINGS_MODULE", - os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.dev"), + os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.platform"), ) +from django.core.asgi import get_asgi_application # noqa: E402 + application = get_asgi_application() diff --git a/backend/backend/celery_service.py b/backend/backend/celery.py similarity index 66% rename from backend/backend/celery_service.py rename to backend/backend/celery.py index 99effcd78..4580a1908 100644 --- a/backend/backend/celery_service.py +++ b/backend/backend/celery.py @@ -2,20 +2,25 @@ import os -from celery import Celery -from django.conf import settings -from utils.constants import ExecutionLogConstants +from dotenv import find_dotenv, load_dotenv -from backend.celery_task import TaskRegistry - -# Set the default Django settings module for the 'celery' program. +# NOTE: +# Do this before loading any environment variables. os.environ.setdefault( "DJANGO_SETTINGS_MODULE", - os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.dev"), + os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.celery"), ) +# Load environment variables. +load_dotenv(find_dotenv() or "") + +from celery import Celery # noqa: E402 +from django.conf import settings # noqa: E402 +from utils.celery.constants import ExecutionLogConstants # noqa: E402 +from utils.celery.task_registry import TaskRegistry # noqa: E402 + # Create a Celery instance. Default time zone is UTC. -app = Celery("backend") +app = Celery("celery") # To register the custom tasks. TaskRegistry() @@ -24,7 +29,6 @@ app.conf.broker_url = settings.CELERY_BROKER_URL app.conf.result_backend = settings.CELERY_RESULT_BACKEND - # Load task modules from all registered Django app configs. app.config_from_object("django.conf:settings", namespace="CELERY") diff --git a/backend/backend/flowerconfig.py b/backend/backend/flowerconfig.py deleted file mode 100644 index 1d25d6458..000000000 --- a/backend/backend/flowerconfig.py +++ /dev/null @@ -1,15 +0,0 @@ -# Flower is a real-time web based monitor and administration tool -# for Celery. It’s under active development, -# but is already an essential tool. -from django.conf import settings - -# Broker URL -BROKER_URL = settings.CELERY_BROKER_URL - -# Flower web port -PORT = 5555 - -# Enable basic authentication (when required) -# basic_auth = { -# 'username': 'password' -# } diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py index d47788b57..59025780c 100644 --- a/backend/backend/settings/base.py +++ b/backend/backend/settings/base.py @@ -9,15 +9,20 @@ https://docs.djangoproject.com/en/4.2/ref/settings/ """ -import os -from pathlib import Path -from typing import Optional -from urllib.parse import quote_plus - from dotenv import find_dotenv, load_dotenv -from backend.constants import FeatureFlag -from unstract.flags.feature_flag import check_feature_flag_status +# Load environment variables. +load_dotenv(find_dotenv() or "") + +import os # noqa: E402 +from pathlib import Path # noqa: E402 +from typing import Optional # noqa: E402 + +from backend.constants import FeatureFlag # noqa: E402 + +# Requires PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python env setting +# to be loaded prior. +from unstract.flags.feature_flag import check_feature_flag_status # noqa: E402 missing_settings = [] @@ -85,13 +90,6 @@ def get_required_setting( }, } - -ENV_FILE = find_dotenv() -if ENV_FILE: - load_dotenv(ENV_FILE) - -# Loading environment variables - WORKFLOW_ACTION_EXPIRATION_TIME_IN_SECOND = os.environ.get( "WORKFLOW_ACTION_EXPIRATION_TIME_IN_SECOND", 10800 ) @@ -163,10 +161,6 @@ def get_required_setting( LOG_HISTORY_CONSUMER_INTERVAL = int( get_required_setting("LOG_HISTORY_CONSUMER_INTERVAL", "60") ) -LOGS_BATCH_LIMIT = int(get_required_setting("LOGS_BATCH_LIMIT", "30")) -CELERY_BROKER_URL = get_required_setting( - "CELERY_BROKER_URL", f"redis://{REDIS_HOST}:{REDIS_PORT}" -) INDEXING_FLAG_TTL = int(get_required_setting("INDEXING_FLAG_TTL")) NOTIFICATION_TIMEOUT = int(get_required_setting("NOTIFICATION_TIMEOUT", "5")) @@ -401,22 +395,10 @@ def get_required_setting( } -# CELERY_RESULT_BACKEND = f"redis://{REDIS_HOST}:{REDIS_PORT}/1" -# Postgres as result backend -CELERY_RESULT_BACKEND = ( - f"db+postgresql://{DB_USER}:{quote_plus(DB_PASSWORD)}" - f"@{DB_HOST}:{DB_PORT}/{DB_NAME}" -) -CELERY_ACCEPT_CONTENT = ["json"] -CELERY_TASK_SERIALIZER = "json" -CELERY_RESULT_SERIALIZER = "json" -CELERY_TIMEZONE = "UTC" -CELERY_TASK_MAX_RETRIES = 3 -CELERY_TASK_RETRY_BACKOFF = 60 # Time in seconds before retrying the task - # Feature Flag FEATURE_FLAG_SERVICE_URL = {"evaluate": f"{FLIPT_BASE_URL}/api/v1/flags/evaluate/"} + # Password validation # https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators diff --git a/backend/backend/settings/celery.py b/backend/backend/settings/celery.py new file mode 100644 index 000000000..a8bce4522 --- /dev/null +++ b/backend/backend/settings/celery.py @@ -0,0 +1,100 @@ +import os +from typing import Any, Optional +from urllib.parse import quote_plus + +from backend.constants import FeatureFlag + +# Requires PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python env setting +# to be loaded prior. +from unstract.flags.feature_flag import check_feature_flag_status + +missing_settings = [] + + +def get_required_setting( + setting_key: str, default: Optional[Any] = None +) -> Optional[str]: + """Get the value of an environment variable specified by the given key. Add + missing keys to `missing_settings` so that exception can be raised at the + end. + + Args: + key (str): The key of the environment variable + default (Optional[str], optional): Default value to return incase of + env not found. Defaults to None. + + Returns: + Optional[str]: The value of the environment variable if found, + otherwise the default value. + """ + data = os.environ.get(setting_key, default) + if not data: + missing_settings.append(setting_key) + return data + + +DB_NAME = os.environ.get("DB_NAME", "unstract_db") +DB_USER = os.environ.get("DB_USER", "unstract_dev") +DB_HOST = os.environ.get("DB_HOST", "backend-db-1") +DB_PASSWORD = os.environ.get("DB_PASSWORD", "unstract_pass") +DB_PORT = os.environ.get("DB_PORT", 5432) +if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2): + DB_ENGINE = "django.db.backends.postgresql" +else: + DB_ENGINE = "django_tenants.postgresql_backend" + + +REDIS_USER = os.environ.get("REDIS_USER", "default") +REDIS_PASSWORD = os.environ.get("REDIS_PASSWORD", "") +REDIS_HOST = os.environ.get("REDIS_HOST", "localhost") +REDIS_PORT = os.environ.get("REDIS_PORT", "6379") +REDIS_DB = os.environ.get("REDIS_DB", "") + +ENABLE_LOG_HISTORY = get_required_setting("ENABLE_LOG_HISTORY") +LOG_HISTORY_CONSUMER_INTERVAL = int( + get_required_setting("LOG_HISTORY_CONSUMER_INTERVAL", "60") +) +LOGS_BATCH_LIMIT = int(get_required_setting("LOGS_BATCH_LIMIT", "30")) + +CORS_ALLOWED_ORIGINS = [ + "http://localhost:3000", + "http://127.0.0.1:3000", + "http://frontend.unstract.localhost", + # Other allowed origins if needed +] + +# Database +# https://docs.djangoproject.com/en/4.2/ref/settings/#databases +DATABASES = { + "default": { + "ENGINE": DB_ENGINE, + "NAME": f"{DB_NAME}", + "USER": f"{DB_USER}", + "HOST": f"{DB_HOST}", + "PASSWORD": f"{DB_PASSWORD}", + "PORT": f"{DB_PORT}", + "ATOMIC_REQUESTS": True, + } +} + +# SocketIO connection manager +SOCKET_IO_MANAGER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}" + +CELERY_BROKER_URL = get_required_setting( + "CELERY_BROKER_URL", f"redis://{REDIS_HOST}:{REDIS_PORT}" +) +# CELERY_RESULT_BACKEND = f"redis://{REDIS_HOST}:{REDIS_PORT}/1" +# Postgres as result backend +CELERY_RESULT_BACKEND = ( + f"db+postgresql://{DB_USER}:{quote_plus(DB_PASSWORD)}" + f"@{DB_HOST}:{DB_PORT}/{DB_NAME}" +) +CELERY_ACCEPT_CONTENT = ["json"] +CELERY_TASK_SERIALIZER = "json" +CELERY_RESULT_SERIALIZER = "json" +CELERY_TIMEZONE = "UTC" +CELERY_TASK_MAX_RETRIES = 3 +CELERY_TASK_RETRY_BACKOFF = 60 # Time in seconds before retrying the task + + +INSTALLED_APPS = ["django.contrib.contenttypes", "django_celery_beat"] diff --git a/backend/backend/settings/dev.py b/backend/backend/settings/platform.py similarity index 97% rename from backend/backend/settings/dev.py rename to backend/backend/settings/platform.py index 8765cd234..f87099c04 100644 --- a/backend/backend/settings/dev.py +++ b/backend/backend/settings/platform.py @@ -1,6 +1,6 @@ from backend.settings.base import * # noqa: F401, F403 -DEBUG = True +DEBUG = False X_FRAME_OPTIONS = "http://localhost:3000" X_FRAME_OPTIONS = "ALLOW-FROM http://localhost:3000" diff --git a/backend/backend/wsgi.py b/backend/backend/wsgi.py index 9a654eb50..b00212a83 100644 --- a/backend/backend/wsgi.py +++ b/backend/backend/wsgi.py @@ -8,18 +8,19 @@ import os -from django.conf import settings -from django.core.wsgi import get_wsgi_application -from dotenv import load_dotenv -from utils.log_events import start_server +from dotenv import find_dotenv, load_dotenv -load_dotenv() +load_dotenv(find_dotenv() or "") os.environ.setdefault( "DJANGO_SETTINGS_MODULE", - os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.dev"), + os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.platform"), ) +from django.conf import settings # noqa: E402 +from django.core.wsgi import get_wsgi_application # noqa: E402 +from utils.log_events import start_server # noqa: E402 + django_app = get_wsgi_application() application = start_server(django_app, f"{settings.PATH_PREFIX}/socket") diff --git a/backend/manage.py b/backend/manage.py index c5d2f5890..bb3de315f 100755 --- a/backend/manage.py +++ b/backend/manage.py @@ -12,7 +12,7 @@ def main() -> None: load_dotenv() os.environ.setdefault( "DJANGO_SETTINGS_MODULE", - os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.dev"), + os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.platform"), ) try: from django.core.management import execute_from_command_line diff --git a/backend/sample.env b/backend/sample.env index bb8267dfd..fd225216c 100644 --- a/backend/sample.env +++ b/backend/sample.env @@ -1,4 +1,4 @@ -DJANGO_SETTINGS_MODULE='backend.settings.dev' +DJANGO_SETTINGS_MODULE='backend.settings.platform' # Default log level DEFAULT_LOG_LEVEL="INFO" @@ -97,6 +97,8 @@ STRUCTURE_TOOL_IMAGE_TAG="0.0.36" # Feature Flags EVALUATION_SERVER_IP=unstract-flipt EVALUATION_SERVER_PORT=9000 +EVALUATION_SERVER_WARNINGS="false" + PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python diff --git a/backend/utils/celery/constants.py b/backend/utils/celery/constants.py new file mode 100644 index 000000000..8562bcee9 --- /dev/null +++ b/backend/utils/celery/constants.py @@ -0,0 +1,19 @@ +from django.conf import settings +from utils.common_utils import CommonUtils + + +class ExecutionLogConstants: + """Constants for ExecutionLog. + + Attributes: + IS_ENABLED (bool): Whether to enable log history. + LOGS_BATCH_LIMIT (str): The maximum number of logs to store in a batch. + LOG_QUEUE_NAME (str): The name of the queue to store log history. + CELERY_QUEUE_NAME (str): The name of the Celery queue to schedule log + history consumers. + """ + + IS_ENABLED: bool = CommonUtils.str_to_bool(settings.ENABLE_LOG_HISTORY) + LOGS_BATCH_LIMIT: int = settings.LOGS_BATCH_LIMIT + LOG_QUEUE_NAME: str = "log_history_queue" + CELERY_QUEUE_NAME = "celery_periodic_logs" diff --git a/backend/backend/celery_task.py b/backend/utils/celery/task_registry.py similarity index 100% rename from backend/backend/celery_task.py rename to backend/utils/celery/task_registry.py diff --git a/backend/utils/constants.py b/backend/utils/constants.py index d431dbbb1..6f9759edb 100644 --- a/backend/utils/constants.py +++ b/backend/utils/constants.py @@ -1,13 +1,6 @@ -import os - from django.conf import settings from utils.common_utils import CommonUtils -os.environ.setdefault( - "DJANGO_SETTINGS_MODULE", - os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.dev"), -) - class Account: CREATED_BY = "created_by" @@ -50,7 +43,6 @@ class ExecutionLogConstants: CONSUMER_INTERVAL (int): The interval (in seconds) between log history consumers. LOG_QUEUE_NAME (str): The name of the queue to store log history. - LOGS_BATCH_LIMIT (str): The maximum number of logs to store in a batch. CELERY_QUEUE_NAME (str): The name of the Celery queue to schedule log history consumers. PERIODIC_TASK_NAME (str): The name of the Celery periodic task to schedule @@ -60,7 +52,6 @@ class ExecutionLogConstants: IS_ENABLED: bool = CommonUtils.str_to_bool(settings.ENABLE_LOG_HISTORY) CONSUMER_INTERVAL: int = settings.LOG_HISTORY_CONSUMER_INTERVAL - LOGS_BATCH_LIMIT: int = settings.LOGS_BATCH_LIMIT LOG_QUEUE_NAME: str = "log_history_queue" CELERY_QUEUE_NAME = "celery_periodic_logs" PERIODIC_TASK_NAME = "workflow_log_history" diff --git a/dev-env-cli.sh b/dev-env-cli.sh index a0826c159..c9f14dc1b 100755 --- a/dev-env-cli.sh +++ b/dev-env-cli.sh @@ -167,6 +167,14 @@ parse_args() { debug "OPTION verbose: $opt_verbose" } +check_for_active_venv() { + if [[ -v VIRTUAL_ENV ]] && [[ -n "$VIRTUAL_ENV" ]]; then + echo -e "$red_text""Detected active virtual env from: $VIRTUAL_ENV.""$default_text" + echo -e "$red_text""Please run 'deactivate' first. Exiting.""$default_text" + exit 1 + fi +} + setup_venv() { if [ "$opt_setup_venv" = false ]; then return @@ -288,6 +296,7 @@ run_pre_commit_hook() { # Run Unstract platform - BEGIN # check_dependencies +check_for_active_venv opt_setup_venv=false opt_activate_venv=false diff --git a/unstract/flags/README.md b/unstract/flags/README.md index 8bcfe6e6c..52f131fa4 100644 --- a/unstract/flags/README.md +++ b/unstract/flags/README.md @@ -1,27 +1,14 @@ # FEATURE FLAGS -## Prerequist knowlege: - To effectively use this repository, it's beneficial to have a foundational understanding of basic gRPC functionality and experience working with gRPC in Python. Familiarity with concepts such as gRPC services, protocol buffers, and the structure of gRPC-based applications will enhance your ability to leverage the tools and code provided here. +## Prerequisites - -""" -This code demonstrates the usage of feature flags with the Flipt service running as a Docker container. +To effectively use this repository, it's beneficial to have a foundational understanding of basic gRPC functionality and experience working with gRPC in Python. Familiarity with concepts such as gRPC services, protocol buffers, and the structure of gRPC-based applications will enhance your ability to leverage the tools and code provided here. Feature flags are a software development technique that allows you to enable or disable certain features in your application without deploying new code. Flipt is a feature flagging service that provides a centralized way to manage and control feature flags. -In this code, we are using Flipt to check the status of a feature flag before executing a specific block of code. The Flipt service is assumed to be running as a Docker container. - -To use this code, make sure you have the Flipt service running as a Docker container and configure the necessary connection details in the code. Then, you can use the `is_feature_enabled` function to check the status of a feature flag before executing the corresponding code. - -Note: This code assumes that you have already set up the necessary feature flags and configurations in the Flipt service. - -For more information on feature flags and Flipt, refer to the documentation: - -- Feature flags: https://en.wikipedia.org/wiki/Feature_toggle -- Flipt: https://flipt.io/ - -""" +The following code demonstrates the usage of feature flags with the Flipt service running as a Docker container. +```python def is_feature_enabled(feature_flag: str) -> bool: """ Check if a feature flag is enabled. @@ -42,31 +29,44 @@ if is_feature_enabled("my_feature"): else: # Execute code for disabled feature pass +``` -## Feature flags in Pandora +In this code, we are using Flipt to check the status of a feature flag before executing a specific block of code. The Flipt service is assumed to be running as a Docker container. -- Refer related files in /backend/feature_flag and /unstract/flags +To use this code, make sure you have the Flipt service running as a Docker container and configure the necessary connection details in the code. Then, you can use the `is_feature_enabled` function to check the status of a feature flag before executing the corresponding code. -- Set required variables in backend .env to utilize feature flags: +Note: This code assumes that you have already set up the necessary feature flags and configurations in the Flipt service. - EVALUATION_SERVER_IP= - EVALUATION_SERVER_PORT= +For more information on feature flags and Flipt, refer to the documentation: + +- Feature flags: https://en.wikipedia.org/wiki/Feature_toggle +- Flipt: https://flipt.io/ + +## Feature flags in Unstract - These variables has to set the FLIPT server values. +- Refer related files in `/backend/feature_flag` and `/unstract/flags` -- Generate python files for gRPC using .proto file inside feature_flags app with protoc - The generated files have suffix with pb, indicating proto buffer files +- Set required variables in backend `.env` to utilize feature flags: + ```bash + EVALUATION_SERVER_IP= + EVALUATION_SERVER_PORT= + EVALUATION_SERVER_WARNINGS="true|false" ``` - + +- Generate python files for gRPC using `.proto` file inside `feature_flags` app with `protoc`. + The generated files have prefix `_pb`, indicating protocol buffers code. + + ```bash python -m grpc_tools.protoc -I protos --python_out=. --grpc_python_out=. protos/evaluation.proto protoc --python_out=./protos protos/evaluation.proto - ``` -## Explanation: - The first command (python -m grpc_tools.protoc ...) generates the gRPC Python files (_pb2.py and _pb2_grpc.py) based on the evaluation.proto file located in the protos directory. - The second command (protoc --python_out=./protos ...) generates the standard Python files (_pb2.py) based on the same evaluation.proto file. +## Explanation + +The first command (`python -m grpc_tools.protoc ...`) generates the gRPC Python files (`_pb2.py` and `_pb2_grpc.py`) based on the `evaluation.proto` file located in the `protos` directory. + +The second command (`protoc --python_out=./protos ...`) generates the standard Python files (`_pb2.py`) based on the same `evaluation.proto` file. - After running the compilation commands, you can use the generated files in your Python project to implement gRPC client and server functionality. Refer to the generated files for the specific classes and methods available for use. +After running the compilation commands, you can use the generated files in your Python project to implement gRPC client and server functionality. Refer to the generated files for the specific classes and methods available for use. diff --git a/unstract/flags/src/unstract/flags/client/base.py b/unstract/flags/src/unstract/flags/client/base.py index 8ecf88fa2..548d7347d 100644 --- a/unstract/flags/src/unstract/flags/client/base.py +++ b/unstract/flags/src/unstract/flags/client/base.py @@ -7,6 +7,9 @@ class BaseClient: def __init__(self, stub_class) -> None: evaluation_server_ip = os.environ.get("EVALUATION_SERVER_IP", "") evaluation_server_port = os.environ.get("EVALUATION_SERVER_PORT", "") + evaluation_server_warnings = os.environ.get( + "EVALUATION_SERVER_WARNINGS", "false" + ) if not evaluation_server_ip: raise ValueError("No response from server, refer README.md.") @@ -15,3 +18,4 @@ def __init__(self, stub_class) -> None: f"{evaluation_server_ip}:{evaluation_server_port}" ) self.stub = stub_class(self.channel) + self.warnings = evaluation_server_warnings.lower() == "true" diff --git a/unstract/flags/src/unstract/flags/client/evaluation.py b/unstract/flags/src/unstract/flags/client/evaluation.py index 537abe869..6b9d2de17 100644 --- a/unstract/flags/src/unstract/flags/client/evaluation.py +++ b/unstract/flags/src/unstract/flags/client/evaluation.py @@ -52,14 +52,17 @@ def boolean_evaluate_feature_flag( return bool(response.enabled) except grpc.RpcError as e: if e.code() == grpc.StatusCode.NOT_FOUND: - logger.warning( - f"Flag key {flag_key} not found in namespace {namespace_key}." - ) + if self.warnings: + logger.warning( + f"Flag key {flag_key} not found in namespace {namespace_key}." + ) elif e.code() == grpc.StatusCode.UNAVAILABLE: - logger.warning(f"Evaluation server is unavailable: {e.details()}.") + if self.warnings: + logger.warning(f"Evaluation server is unavailable: {e.details()}.") else: - logger.warning( - f"Error evaluating feature flag {flag_key} for {namespace_key}" - f" : {str(e)}" - ) + if self.warnings: + logger.warning( + f"Error evaluating feature flag {flag_key} for {namespace_key}" + f" : {str(e)}" + ) return False diff --git a/unstract/flags/src/unstract/flags/sample.env b/unstract/flags/src/unstract/flags/sample.env index 3c531857c..23ec248ba 100644 --- a/unstract/flags/src/unstract/flags/sample.env +++ b/unstract/flags/src/unstract/flags/sample.env @@ -1,2 +1,3 @@ -FEATURE_FLAG_SERVICE_HOST=0.0.0.0 -flipt_server_port=9000 +EVALUATION_SERVER_IP=unstract-flipt +EVALUATION_SERVER_PORT=9005 +EVALUATION_SERVER_WARNINGS="false" From 47d08fe3e66755fa9a7f81160cc7c01fa33df7f0 Mon Sep 17 00:00:00 2001 From: Hari John Kuriakose Date: Fri, 16 Aug 2024 15:52:25 +0530 Subject: [PATCH 2/6] * Add .celery.env * Allow building docker images locally from current branch * Cleanup run platform script * Update pdm lock files for backend and prompt service --- backend/.celery.env | 30 +++++ backend/backend/settings/celery.py | 2 +- backend/pdm.lock | 176 ++++++++++++++++------------- backend/sample.celery.env | 30 +++++ docker/docker-compose.yaml | 6 +- prompt-service/pdm.lock | 155 +++++++++++++------------ run-platform.sh | 70 ++++++------ 7 files changed, 268 insertions(+), 201 deletions(-) create mode 100644 backend/.celery.env create mode 100644 backend/sample.celery.env diff --git a/backend/.celery.env b/backend/.celery.env new file mode 100644 index 000000000..832c9c3a3 --- /dev/null +++ b/backend/.celery.env @@ -0,0 +1,30 @@ +DJANGO_SETTINGS_MODULE='backend.settings.celery' + +# Default log level +DEFAULT_LOG_LEVEL="INFO" + +# Postgres DB envs +DB_HOST='unstract-db' +DB_USER='unstract_dev' +DB_PASSWORD='unstract_pass' +DB_NAME='unstract_db' +DB_PORT=5432 + +# Redis +REDIS_HOST="unstract-redis" +REDIS_PORT=6379 +REDIS_PASSWORD="" +REDIS_USER=default + +# Protocol buffers generated code. +PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python + +# Enable logging of workflow history. +ENABLE_LOG_HISTORY=True +# Interval in seconds for periodic consumer operations. +LOG_HISTORY_CONSUMER_INTERVAL=30 +# Maximum number of logs to insert in a single batch. +LOGS_BATCH_LIMIT=30 + +# Celery Configuration +CELERY_BROKER_URL="redis://unstract-redis:6379" diff --git a/backend/backend/settings/celery.py b/backend/backend/settings/celery.py index a8bce4522..147a93838 100644 --- a/backend/backend/settings/celery.py +++ b/backend/backend/settings/celery.py @@ -46,7 +46,7 @@ def get_required_setting( REDIS_USER = os.environ.get("REDIS_USER", "default") REDIS_PASSWORD = os.environ.get("REDIS_PASSWORD", "") -REDIS_HOST = os.environ.get("REDIS_HOST", "localhost") +REDIS_HOST = os.environ.get("REDIS_HOST", "unstract-redis") REDIS_PORT = os.environ.get("REDIS_PORT", "6379") REDIS_DB = os.environ.get("REDIS_DB", "") diff --git a/backend/pdm.lock b/backend/pdm.lock index 136d9eb71..0b8df0270 100644 --- a/backend/pdm.lock +++ b/backend/pdm.lock @@ -3,10 +3,13 @@ [metadata] groups = ["default", "deploy", "dev", "test"] -strategy = ["cross_platform", "inherit_metadata"] -lock_version = "4.4.2" +strategy = ["inherit_metadata"] +lock_version = "4.5.0" content_hash = "sha256:00e0cde5d7b98e51171b22f703249f26104a820e4d5fd367d08296706e4600d9" +[[metadata.targets]] +requires_python = ">=3.9,<3.11.1" + [[package]] name = "adlfs" version = "2023.8.0" @@ -61,13 +64,13 @@ files = [ [[package]] name = "aiohappyeyeballs" -version = "2.3.5" +version = "2.3.6" requires_python = ">=3.8" summary = "Happy Eyeballs for asyncio" groups = ["default", "dev"] files = [ - {file = "aiohappyeyeballs-2.3.5-py3-none-any.whl", hash = "sha256:4d6dea59215537dbc746e93e779caea8178c866856a721c9c660d7a5a7b8be03"}, - {file = "aiohappyeyeballs-2.3.5.tar.gz", hash = "sha256:6fa48b9f1317254f122a07a131a86b71ca6946ca989ce6326fff54a99a920105"}, + {file = "aiohappyeyeballs-2.3.6-py3-none-any.whl", hash = "sha256:15dca2611fa78442f1cb54cf07ffb998573f2b4fbeab45ca8554c045665c896b"}, + {file = "aiohappyeyeballs-2.3.6.tar.gz", hash = "sha256:88211068d2a40e0436033956d7de3926ff36d54776f8b1022d6b21320cadae79"}, ] [[package]] @@ -182,6 +185,9 @@ version = "0.7.0" requires_python = ">=3.8" summary = "Reusable constraint types to use with typing.Annotated" groups = ["default", "dev"] +dependencies = [ + "typing-extensions>=4.0.0; python_version < \"3.9\"", +] files = [ {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, @@ -195,6 +201,7 @@ summary = "The official Python library for the anthropic API" groups = ["default", "dev"] dependencies = [ "anyio<5,>=3.5.0", + "cached-property; python_version < \"3.8\"", "distro<2,>=1.7.0", "httpx<1,>=0.23.0", "jiter<1,>=0.4.0", @@ -266,6 +273,9 @@ requires_python = ">=3.7" summary = "Timeout context manager for asyncio programs" groups = ["default", "dev"] marker = "python_version < \"3.12.0\"" +dependencies = [ + "typing-extensions>=3.6.5; python_version < \"3.8\"", +] files = [ {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, @@ -314,6 +324,9 @@ version = "24.2.0" requires_python = ">=3.7" summary = "Classes Without Boilerplate" groups = ["default", "dev"] +dependencies = [ + "importlib-metadata; python_version < \"3.8\"", +] files = [ {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"}, {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"}, @@ -337,6 +350,9 @@ name = "azure-common" version = "1.1.28" summary = "Microsoft Azure Client Library for Python (Common)" groups = ["default"] +dependencies = [ + "azure-nspkg; python_version < \"3.0\"", +] files = [ {file = "azure-common-1.1.28.zip", hash = "sha256:4ac0cd3214e36b6a1b6a442686722a5d8cc449603aa833f3f0f40bda836704a3"}, {file = "azure_common-1.1.28-py2.py3-none-any.whl", hash = "sha256:5c12d3dcf4ec20599ca6b0d3e09e86e146353d443e7fcc050c9a19c1f9df20ad"}, @@ -364,8 +380,11 @@ version = "0.0.53" summary = "Azure Data Lake Store Filesystem Client Library for Python" groups = ["default", "dev"] dependencies = [ + "azure-nspkg; python_version < \"3.0\"", "cffi", + "futures; python_version <= \"2.7\"", "msal<2,>=1.16.0", + "pathlib2; python_version < \"3.4\"", "requests>=2.20.0", ] files = [ @@ -572,11 +591,13 @@ requires_python = ">=3.8" summary = "Distributed Task Queue." groups = ["default"] dependencies = [ + "backports-zoneinfo>=0.2.1; python_version < \"3.9\"", "billiard<5.0,>=4.2.0", "click-didyoumean>=0.3.0", "click-plugins>=1.1.1", "click-repl>=0.2.0", "click<9.0,>=8.1.2", + "importlib-metadata>=3.6; python_version < \"3.8\"", "kombu<6.0,>=5.3.4", "python-dateutil>=2.8.2", "tzdata>=2022.7", @@ -711,6 +732,7 @@ summary = "Composable command line interface toolkit" groups = ["default", "dev"] dependencies = [ "colorama; platform_system == \"Windows\"", + "importlib-metadata; python_version < \"3.8\"", ] files = [ {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, @@ -819,10 +841,6 @@ files = [ {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:b640981bf64a3e978a56167594a0e97db71c89a479da8e175d8bb5be5178c003"}, {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e3114da6d7f95d2dee7d3f4eec16dacff819740bbab931aff8648cb13c5ff5e7"}, {file = "cryptography-41.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d5ec85080cce7b0513cfd233914eb8b7bbd0633f1d1703aa28d1dd5a72f678ec"}, - {file = "cryptography-41.0.7-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7a698cb1dac82c35fcf8fe3417a3aaba97de16a01ac914b89a0889d364d2f6be"}, - {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:37a138589b12069efb424220bf78eac59ca68b95696fc622b6ccc1c0a197204a"}, - {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:68a2dec79deebc5d26d617bfdf6e8aab065a4f34934b22d3b5010df3ba36612c"}, - {file = "cryptography-41.0.7-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:09616eeaef406f99046553b8a40fbf8b1e70795a91885ba4c96a70793de5504a"}, {file = "cryptography-41.0.7-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:48a0476626da912a44cc078f9893f292f0b3e4c739caf289268168d8f4702a39"}, {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c7f3201ec47d5207841402594f1d7950879ef890c0c495052fa62f58283fde1a"}, {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c5ca78485a255e03c32b513f8c2bc39fedb7f5c5f8535545bdc223a03b24f248"}, @@ -910,6 +928,7 @@ summary = "A high-level Python web framework that encourages rapid development a groups = ["default"] dependencies = [ "asgiref<4,>=3.6.0", + "backports-zoneinfo; python_version < \"3.9\"", "sqlparse>=0.3.1", "tzdata; sys_platform == \"win32\"", ] @@ -925,9 +944,11 @@ summary = "Database-backed Periodic Tasks." groups = ["default"] dependencies = [ "Django<5.0,>=2.2", + "backports-zoneinfo; python_version < \"3.9\"", "celery<6.0,>=5.2.3", "cron-descriptor>=1.2.32", "django-timezone-field>=5.0", + "importlib-metadata<5.0; python_version < \"3.8\"", "python-crontab>=2.3.4", "tzdata", ] @@ -999,6 +1020,7 @@ summary = "A Django app providing DB, form, and REST framework fields for zonein groups = ["default"] dependencies = [ "Django<6.0,>=3.2", + "backports-zoneinfo<0.3.0,>=0.2.1; python_version < \"3.9\"", ] files = [ {file = "django_timezone_field-7.0-py3-none-any.whl", hash = "sha256:3232e7ecde66ba4464abb6f9e6b8cc739b914efb9b29dc2cf2eee451f7cc2acb"}, @@ -1332,7 +1354,7 @@ files = [ [[package]] name = "google-api-python-client" -version = "2.140.0" +version = "2.141.0" requires_python = ">=3.7" summary = "Google API Client Library for Python" groups = ["default", "dev"] @@ -1344,8 +1366,8 @@ dependencies = [ "uritemplate<5,>=3.0.1", ] files = [ - {file = "google_api_python_client-2.140.0-py2.py3-none-any.whl", hash = "sha256:aeb4bb99e9fdd241473da5ff35464a0658fea0db76fe89c0f8c77ecfc3813404"}, - {file = "google_api_python_client-2.140.0.tar.gz", hash = "sha256:0bb973adccbe66a3d0a70abe4e49b3f2f004d849416bfec38d22b75649d389d8"}, + {file = "google_api_python_client-2.141.0-py2.py3-none-any.whl", hash = "sha256:43c05322b91791204465291b3852718fae38d4f84b411d8be847c4f86882652a"}, + {file = "google_api_python_client-2.141.0.tar.gz", hash = "sha256:0f225b1f45d5a6f8c2a400f48729f5d6da9a81138e81e0478d61fdd8edf6563a"}, ] [[package]] @@ -1397,7 +1419,7 @@ files = [ [[package]] name = "google-cloud-aiplatform" -version = "1.61.0" +version = "1.62.0" requires_python = ">=3.8" summary = "Vertex AI API client library" groups = ["default", "dev"] @@ -1415,8 +1437,8 @@ dependencies = [ "shapely<3.0.0dev", ] files = [ - {file = "google-cloud-aiplatform-1.61.0.tar.gz", hash = "sha256:648e3cd7bb75be706d3c31d852a3d4d8a2e616ad4db4cf520ef4430615cf8ad9"}, - {file = "google_cloud_aiplatform-1.61.0-py2.py3-none-any.whl", hash = "sha256:57b36d5fa085e68197e9fc576c43263a7cad320483aa3b166bcd1fdc7e8f49e7"}, + {file = "google-cloud-aiplatform-1.62.0.tar.gz", hash = "sha256:e15d5b2a99e30d4a16f4c51cfb8129962e6da41a9027d2ea696abe0e2f006fe8"}, + {file = "google_cloud_aiplatform-1.62.0-py2.py3-none-any.whl", hash = "sha256:d7738e0fd4494a54ae08a51755a2143d58937cba2db826189771f45566c9ee3c"}, ] [[package]] @@ -1451,6 +1473,7 @@ groups = ["default", "dev"] dependencies = [ "google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.6", "google-auth<3.0dev,>=1.25.0", + "importlib-metadata>1.0.0; python_version < \"3.8\"", ] files = [ {file = "google-cloud-core-2.4.1.tar.gz", hash = "sha256:9b7749272a812bde58fff28868d0c5e2f585b82f37e09a1f6ed2d4d10f134073"}, @@ -1550,16 +1573,6 @@ files = [ {file = "google_crc32c-1.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7c074fece789b5034b9b1404a1f8208fc2d4c6ce9decdd16e8220c5a793e6f61"}, {file = "google_crc32c-1.5.0-cp39-cp39-win32.whl", hash = "sha256:7f57f14606cd1dd0f0de396e1e53824c371e9544a822648cd76c034d209b559c"}, {file = "google_crc32c-1.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:a2355cba1f4ad8b6988a4ca3feed5bff33f6af2d7f134852cf279c2aebfde541"}, - {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:f314013e7dcd5cf45ab1945d92e713eec788166262ae8deb2cfacd53def27325"}, - {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b747a674c20a67343cb61d43fdd9207ce5da6a99f629c6e2541aa0e89215bcd"}, - {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8f24ed114432de109aa9fd317278518a5af2d31ac2ea6b952b2f7782b43da091"}, - {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8667b48e7a7ef66afba2c81e1094ef526388d35b873966d8a9a447974ed9178"}, - {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:1c7abdac90433b09bad6c43a43af253e688c9cfc1c86d332aed13f9a7c7f65e2"}, - {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:6f998db4e71b645350b9ac28a2167e6632c239963ca9da411523bb439c5c514d"}, - {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c99616c853bb585301df6de07ca2cadad344fd1ada6d62bb30aec05219c45d2"}, - {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ad40e31093a4af319dadf503b2467ccdc8f67c72e4bcba97f8c10cb078207b5"}, - {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd67cf24a553339d5062eff51013780a00d6f97a39ca062781d06b3a73b15462"}, - {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:398af5e3ba9cf768787eef45c803ff9614cc3e22a5b2f7d7ae116df8b11e3314"}, {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:b1f8133c9a275df5613a451e73f36c2aea4fe13c5c8997e22cf355ebd7bd0728"}, {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ba053c5f50430a3fcfd36f75aff9caeba0440b2d076afdb79a318d6ca245f88"}, {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:272d3892a1e1a2dbc39cc5cde96834c236d5327e2122d3aaa19f6614531bb6eb"}, @@ -1801,6 +1814,7 @@ requires_python = ">=3.7" summary = "WSGI HTTP Server for UNIX" groups = ["deploy"] dependencies = [ + "importlib-metadata; python_version < \"3.8\"", "packaging", ] files = [ @@ -1814,6 +1828,9 @@ version = "0.14.0" requires_python = ">=3.7" summary = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" groups = ["default", "dev"] +dependencies = [ + "typing-extensions; python_version < \"3.8\"", +] files = [ {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, @@ -1868,6 +1885,7 @@ summary = "A comprehensive HTTP client library." groups = ["default", "dev"] dependencies = [ "pyparsing!=3.0.0,!=3.0.1,!=3.0.2,!=3.0.3,<4,>=2.4.2; python_version > \"3.0\"", + "pyparsing<3,>=2.4.2; python_version < \"3.0\"", ] files = [ {file = "httplib2-0.22.0-py3-none-any.whl", hash = "sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc"}, @@ -2072,7 +2090,9 @@ summary = "An implementation of JSON Schema validation for Python" groups = ["default", "dev"] dependencies = [ "attrs>=22.2.0", + "importlib-resources>=1.4.0; python_version < \"3.9\"", "jsonschema-specifications>=2023.03.6", + "pkgutil-resolve-name>=1.3.10; python_version < \"3.9\"", "referencing>=0.28.4", "rpds-py>=0.7.1", ] @@ -2088,6 +2108,7 @@ requires_python = ">=3.8" summary = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" groups = ["default", "dev"] dependencies = [ + "importlib-resources>=1.4.0; python_version < \"3.9\"", "referencing>=0.31.0", ] files = [ @@ -2103,6 +2124,7 @@ summary = "Messaging library for Python." groups = ["default", "dev"] dependencies = [ "amqp<6.0.0,>=5.1.1", + "backports-zoneinfo[tzdata]>=0.2.1; python_version < \"3.9\"", "typing-extensions; python_version < \"3.10\"", "vine", ] @@ -2294,7 +2316,7 @@ files = [ [[package]] name = "llama-index-legacy" -version = "0.9.48.post1" +version = "0.9.48.post2" requires_python = "<4.0,>=3.8.1" summary = "Interface between LLMs and your data" groups = ["default", "dev"] @@ -2308,7 +2330,7 @@ dependencies = [ "httpx", "nest-asyncio<2.0.0,>=1.5.8", "networkx>=3.0", - "nltk>=3.8.2", + "nltk>=3.8.1", "numpy", "openai>=1.1.0", "pandas", @@ -2319,8 +2341,8 @@ dependencies = [ "typing-inspect>=0.8.0", ] files = [ - {file = "llama_index_legacy-0.9.48.post1-py3-none-any.whl", hash = "sha256:583296162385010ebf92d2a612dd0a504575c04dc1638323bb455b1521aabe57"}, - {file = "llama_index_legacy-0.9.48.post1.tar.gz", hash = "sha256:e8b1603929433fd0cf3287ed700714078534dd202c97bcdbcc83ec3741bb0868"}, + {file = "llama_index_legacy-0.9.48.post2-py3-none-any.whl", hash = "sha256:2581af680a4e577d4f0accd76e8286c5f1054f28a2fb0e8e5758f09ce5da0176"}, + {file = "llama_index_legacy-0.9.48.post2.tar.gz", hash = "sha256:a4c1f10b4d19d005674195c449f4e859022c65c816dcba1a619ef5df922aa212"}, ] [[package]] @@ -2663,7 +2685,7 @@ files = [ [[package]] name = "milvus-lite" -version = "2.4.8" +version = "2.4.9" requires_python = ">=3.7" summary = "A lightweight version of Milvus wrapped with Python." groups = ["default", "dev"] @@ -2672,10 +2694,10 @@ dependencies = [ "tqdm", ] files = [ - {file = "milvus_lite-2.4.8-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:b7e90b34b214884cd44cdc112ab243d4cb197b775498355e2437b6cafea025fe"}, - {file = "milvus_lite-2.4.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:519dfc62709d8f642d98a1c5b1dcde7080d107e6e312d677fef5a3412a40ac08"}, - {file = "milvus_lite-2.4.8-py3-none-manylinux2014_aarch64.whl", hash = "sha256:b21f36d24cbb0e920b4faad607019bb28c1b2c88b4d04680ac8c7697a4ae8a4d"}, - {file = "milvus_lite-2.4.8-py3-none-manylinux2014_x86_64.whl", hash = "sha256:08332a2b9abfe7c4e1d7926068937e46f8fb81f2707928b7bc02c9dc99cebe41"}, + {file = "milvus_lite-2.4.9-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:d3e617b3d68c09ad656d54bc3d8cc4ef6ef56c54015e1563d4fe4bcec6b7c90a"}, + {file = "milvus_lite-2.4.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:6e7029282d6829b277ebb92f64e2370be72b938e34770e1eb649346bda5d1d7f"}, + {file = "milvus_lite-2.4.9-py3-none-manylinux2014_aarch64.whl", hash = "sha256:9b8e991e4e433596f6a399a165c1a506f823ec9133332e03d7f8a114bff4550d"}, + {file = "milvus_lite-2.4.9-py3-none-manylinux2014_x86_64.whl", hash = "sha256:7f53e674602101cfbcf0a4a59d19eaa139dfd5580639f3040ad73d901f24fc0b"}, ] [[package]] @@ -2834,8 +2856,8 @@ files = [ [[package]] name = "nltk" -version = "3.8.2" -requires_python = ">=3.8" +version = "3.8.1" +requires_python = ">=3.7" summary = "Natural Language Toolkit" groups = ["default", "dev"] dependencies = [ @@ -2845,8 +2867,8 @@ dependencies = [ "tqdm", ] files = [ - {file = "nltk-3.8.2-py3-none-any.whl", hash = "sha256:bae044ae22ebe0b694a87c0012233373209f27d5c76d3572599c842740a62fe0"}, - {file = "nltk-3.8.2.tar.gz", hash = "sha256:9c051aa981c6745894906d5c3aad27417f3d1c10d91eefca50382fc922966f31"}, + {file = "nltk-3.8.1-py3-none-any.whl", hash = "sha256:fd5c9109f976fa86bcadba8f91e47f5e9293bd034474752e92a520f81c93dda5"}, + {file = "nltk-3.8.1.zip", hash = "sha256:1834da3d0682cba4f2cede2f9aad6b0fafb6461ba451db0efb6f9c39798d64d3"}, ] [[package]] @@ -2930,12 +2952,13 @@ files = [ [[package]] name = "openai" -version = "1.40.6" +version = "1.40.8" requires_python = ">=3.7.1" summary = "The official Python library for the openai API" groups = ["default", "dev"] dependencies = [ "anyio<5,>=3.5.0", + "cached-property; python_version < \"3.8\"", "distro<2,>=1.7.0", "httpx<1,>=0.23.0", "jiter<1,>=0.4.0", @@ -2945,8 +2968,8 @@ dependencies = [ "typing-extensions<5,>=4.11", ] files = [ - {file = "openai-1.40.6-py3-none-any.whl", hash = "sha256:b36372124a779381a420a34dd96f762baa748b6bdfaf83a6b9f2745f72ccc1c5"}, - {file = "openai-1.40.6.tar.gz", hash = "sha256:2239232bcb7f4bd4ce8e02544b5769618582411cf399816d96686d1b6c1e5c8d"}, + {file = "openai-1.40.8-py3-none-any.whl", hash = "sha256:3ed4ddad48e0dde059c9b4d3dc240e47781beca2811e52ba449ddc4a471a2fd4"}, + {file = "openai-1.40.8.tar.gz", hash = "sha256:e225f830b946378e214c5b2cfa8df28ba2aeb7e9d44f738cb2a926fd971f5bc0"}, ] [[package]] @@ -3009,6 +3032,7 @@ groups = ["default", "dev"] dependencies = [ "numpy<2,>=1.22.4; python_version < \"3.11\"", "numpy<2,>=1.23.2; python_version == \"3.11\"", + "numpy<2,>=1.26.0; python_version >= \"3.12\"", "python-dateutil>=2.8.2", "pytz>=2020.1", "tzdata>=2022.1", @@ -3044,6 +3068,8 @@ groups = ["default", "dev"] dependencies = [ "charset-normalizer>=2.0.0", "cryptography>=36.0.0", + "importlib-metadata; python_version < \"3.8\"", + "typing-extensions; python_version < \"3.8\"", ] files = [ {file = "pdfminer.six-20231228-py3-none-any.whl", hash = "sha256:e8d3c3310e6fbc1fe414090123ab01351634b4ecb021232206c4c9a8ca3e3b8f"}, @@ -3147,6 +3173,7 @@ dependencies = [ "tqdm>=4.64.1", "typing-extensions>=3.7.4", "urllib3>=1.26.0; python_version >= \"3.8\" and python_version < \"3.12\"", + "urllib3>=1.26.5; python_version ~= \"3.12\"", ] files = [ {file = "pinecone_client-3.2.2-py3-none-any.whl", hash = "sha256:7e492fdda23c73726bc0cb94c689bb950d06fb94e82b701a0c610c2e830db327"}, @@ -3159,6 +3186,9 @@ version = "3.11.0" requires_python = ">=3.7" summary = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." groups = ["default", "dev"] +dependencies = [ + "typing-extensions>=4.7.1; python_version < \"3.8\"", +] files = [ {file = "platformdirs-3.11.0-py3-none-any.whl", hash = "sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e"}, {file = "platformdirs-3.11.0.tar.gz", hash = "sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3"}, @@ -3381,6 +3411,7 @@ groups = ["default", "dev"] dependencies = [ "annotated-types>=0.4.0", "pydantic-core==2.20.1", + "typing-extensions>=4.12.2; python_version >= \"3.13\"", "typing-extensions>=4.6.1; python_version < \"3.13\"", ] files = [ @@ -3526,8 +3557,10 @@ dependencies = [ "environs<=9.5.0", "grpcio<=1.63.0,>=1.49.1", "milvus-lite<2.5.0,>=2.4.0; sys_platform != \"win32\"", + "numpy<1.25.0; python_version <= \"3.8\"", "pandas>=1.2.4", "protobuf>=3.20.0", + "setuptools<70.1; python_version <= \"3.8\"", "setuptools>69", "ujson>=2.0.0", ] @@ -3604,6 +3637,7 @@ requires_python = ">=3.6" summary = "A pure-python PDF library capable of splitting, merging, cropping, and transforming PDF files" groups = ["default", "dev"] dependencies = [ + "dataclasses; python_version < \"3.7\"", "typing-extensions>=4.0; python_version < \"3.11\"", ] files = [ @@ -3830,7 +3864,9 @@ dependencies = [ "grpcio-tools>=1.41.0", "grpcio>=1.41.0", "httpx[http2]>=0.20.0", + "numpy<1.21; python_version < \"3.8\"", "numpy>=1.21; python_version >= \"3.8\" and python_version < \"3.12\"", + "numpy>=1.26; python_version >= \"3.12\"", "portalocker<3.0.0,>=2.7.0", "pydantic>=1.10.8", "urllib3<3,>=1.26.14", @@ -3848,6 +3884,8 @@ summary = "Python client for Redis database and key-value store" groups = ["default", "dev"] dependencies = [ "async-timeout>=4.0.3; python_full_version < \"3.11.3\"", + "importlib-metadata>=1.0; python_version < \"3.8\"", + "typing-extensions; python_version < \"3.8\"", ] files = [ {file = "redis-5.0.8-py3-none-any.whl", hash = "sha256:56134ee08ea909106090934adc36f65c9bcbbaecea5b21ba704ba6fb561f8eb4"}, @@ -4155,19 +4193,6 @@ files = [ {file = "safetensors-0.4.4-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e5c9d86d9b13b18aafa88303e2cd21e677f5da2a14c828d2c460fe513af2e9a5"}, {file = "safetensors-0.4.4-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:43251d7f29a59120a26f5a0d9583b9e112999e500afabcfdcb91606d3c5c89e3"}, {file = "safetensors-0.4.4-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:2c42e9b277513b81cf507e6121c7b432b3235f980cac04f39f435b7902857f91"}, - {file = "safetensors-0.4.4-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3daacc9a4e3f428a84dd56bf31f20b768eb0b204af891ed68e1f06db9edf546f"}, - {file = "safetensors-0.4.4-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:218bbb9b883596715fc9997bb42470bf9f21bb832c3b34c2bf744d6fa8f2bbba"}, - {file = "safetensors-0.4.4-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7bd5efc26b39f7fc82d4ab1d86a7f0644c8e34f3699c33f85bfa9a717a030e1b"}, - {file = "safetensors-0.4.4-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:56ad9776b65d8743f86698a1973292c966cf3abff627efc44ed60e66cc538ddd"}, - {file = "safetensors-0.4.4-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:30f23e6253c5f43a809dea02dc28a9f5fa747735dc819f10c073fe1b605e97d4"}, - {file = "safetensors-0.4.4-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:5512078d00263de6cb04e9d26c9ae17611098f52357fea856213e38dc462f81f"}, - {file = "safetensors-0.4.4-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b96c3d9266439d17f35fc2173111d93afc1162f168e95aed122c1ca517b1f8f1"}, - {file = "safetensors-0.4.4-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:08d464aa72a9a13826946b4fb9094bb4b16554bbea2e069e20bd903289b6ced9"}, - {file = "safetensors-0.4.4-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:210160816d5a36cf41f48f38473b6f70d7bcb4b0527bedf0889cc0b4c3bb07db"}, - {file = "safetensors-0.4.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb276a53717f2bcfb6df0bcf284d8a12069002508d4c1ca715799226024ccd45"}, - {file = "safetensors-0.4.4-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a2c28c6487f17d8db0089e8b2cdc13de859366b94cc6cdc50e1b0a4147b56551"}, - {file = "safetensors-0.4.4-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:7915f0c60e4e6e65d90f136d85dd3b429ae9191c36b380e626064694563dbd9f"}, - {file = "safetensors-0.4.4-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:00eea99ae422fbfa0b46065acbc58b46bfafadfcec179d4b4a32d5c45006af6c"}, {file = "safetensors-0.4.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:bb1ed4fcb0b3c2f3ea2c5767434622fe5d660e5752f21ac2e8d737b1e5e480bb"}, {file = "safetensors-0.4.4-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:73fc9a0a4343188bdb421783e600bfaf81d0793cd4cce6bafb3c2ed567a74cd5"}, {file = "safetensors-0.4.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c37e6b714200824c73ca6eaf007382de76f39466a46e97558b8dc4cf643cfbf"}, @@ -4180,13 +4205,13 @@ files = [ [[package]] name = "setuptools" -version = "72.1.0" +version = "72.2.0" requires_python = ">=3.8" summary = "Easily download, build, install, upgrade, and uninstall Python packages" groups = ["default", "dev"] files = [ - {file = "setuptools-72.1.0-py3-none-any.whl", hash = "sha256:5a03e1860cf56bb6ef48ce186b0e557fdba433237481a9a625176c2831be15d1"}, - {file = "setuptools-72.1.0.tar.gz", hash = "sha256:8d243eff56d095e5817f796ede6ae32941278f542e0f941867cc05ae52b162ec"}, + {file = "setuptools-72.2.0-py3-none-any.whl", hash = "sha256:f11dd94b7bae3a156a95ec151f24e4637fb4fa19c878e4d191bfb8b2d82728c4"}, + {file = "setuptools-72.2.0.tar.gz", hash = "sha256:80aacbf633704e9c8bfa1d99fa5dd4dc59573efcf9e4042c13d3bcef91ac2ef9"}, ] [[package]] @@ -4279,6 +4304,7 @@ dependencies = [ "cryptography<42.0.0,>=3.1.0", "filelock<4,>=3.5", "idna<4,>=2.5", + "importlib-metadata; python_version < \"3.8\"", "packaging", "platformdirs<4.0.0,>=2.6.0", "pyOpenSSL<24.0.0,>=16.2.0", @@ -4387,13 +4413,13 @@ files = [ [[package]] name = "soupsieve" -version = "2.5" +version = "2.6" requires_python = ">=3.8" summary = "A modern CSS selector implementation for Beautiful Soup." groups = ["default", "dev"] files = [ - {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"}, - {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"}, + {file = "soupsieve-2.6-py3-none-any.whl", hash = "sha256:e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9"}, + {file = "soupsieve-2.6.tar.gz", hash = "sha256:e2e68417777af359ec65daac1057404a3c8a5455bb8abc36f1a9866ab1a51abb"}, ] [[package]] @@ -4404,6 +4430,7 @@ summary = "Database Abstraction Library" groups = ["default", "dev"] dependencies = [ "greenlet!=0.4.17; (platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\") and python_version < \"3.13\"", + "importlib-metadata; python_version < \"3.8\"", "typing-extensions>=4.6.0", ] files = [ @@ -4609,19 +4636,6 @@ files = [ {file = "tokenizers-0.15.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67a0fe1e49e60c664915e9fb6b0cb19bac082ab1f309188230e4b2920230edb3"}, {file = "tokenizers-0.15.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:4e022fe65e99230b8fd89ebdfea138c24421f91c1a4f4781a8f5016fd5cdfb4d"}, {file = "tokenizers-0.15.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d857be2df69763362ac699f8b251a8cd3fac9d21893de129bc788f8baaef2693"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:708bb3e4283177236309e698da5fcd0879ce8fd37457d7c266d16b550bcbbd18"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:64c35e09e9899b72a76e762f9854e8750213f67567787d45f37ce06daf57ca78"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1257f4394be0d3b00de8c9e840ca5601d0a4a8438361ce9c2b05c7d25f6057b"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02272fe48280e0293a04245ca5d919b2c94a48b408b55e858feae9618138aeda"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:dc3ad9ebc76eabe8b1d7c04d38be884b8f9d60c0cdc09b0aa4e3bcf746de0388"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:32e16bdeffa7c4f46bf2152172ca511808b952701d13e7c18833c0b73cb5c23f"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:fb16ba563d59003028b678d2361a27f7e4ae0ab29c7a80690efa20d829c81fdb"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:2277c36d2d6cdb7876c274547921a42425b6810d38354327dd65a8009acf870c"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1cf75d32e8d250781940d07f7eece253f2fe9ecdb1dc7ba6e3833fa17b82fcbc"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1b3b31884dc8e9b21508bb76da80ebf7308fdb947a17affce815665d5c4d028"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b10122d8d8e30afb43bb1fe21a3619f62c3e2574bff2699cf8af8b0b6c5dc4a3"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d88b96ff0fe8e91f6ef01ba50b0d71db5017fa4e3b1d99681cec89a85faf7bf7"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:37aaec5a52e959892870a7c47cef80c53797c0db9149d458460f4f31e2fb250e"}, {file = "tokenizers-0.15.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e2ea752f2b0fe96eb6e2f3adbbf4d72aaa1272079b0dfa1145507bd6a5d537e6"}, {file = "tokenizers-0.15.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:4b19a808d8799fda23504a5cd31d2f58e6f52f140380082b352f877017d6342b"}, {file = "tokenizers-0.15.2-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:64c86e5e068ac8b19204419ed8ca90f9d25db20578f5881e337d203b314f4104"}, @@ -4646,13 +4660,13 @@ files = [ [[package]] name = "tomlkit" -version = "0.13.0" +version = "0.13.2" requires_python = ">=3.8" summary = "Style preserving TOML library" groups = ["default", "dev"] files = [ - {file = "tomlkit-0.13.0-py3-none-any.whl", hash = "sha256:7075d3042d03b80f603482d69bf0c8f345c2b30e41699fd8883227f89972b264"}, - {file = "tomlkit-0.13.0.tar.gz", hash = "sha256:08ad192699734149f5b97b45f1f18dad7eb1b6d16bc72ad0c2335772650d7b72"}, + {file = "tomlkit-0.13.2-py3-none-any.whl", hash = "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde"}, + {file = "tomlkit-0.13.2.tar.gz", hash = "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79"}, ] [[package]] @@ -4731,6 +4745,7 @@ groups = ["default", "dev"] dependencies = [ "mypy-extensions>=0.3.0", "typing-extensions>=3.7.4", + "typing>=3.7.4; python_version < \"3.5\"", ] files = [ {file = "typing_inspect-0.9.0-py3-none-any.whl", hash = "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f"}, @@ -4791,11 +4806,6 @@ files = [ {file = "ujson-5.10.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fbd8fd427f57a03cff3ad6574b5e299131585d9727c8c366da4624a9069ed746"}, {file = "ujson-5.10.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:beeaf1c48e32f07d8820c705ff8e645f8afa690cca1544adba4ebfa067efdc88"}, {file = "ujson-5.10.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:baed37ea46d756aca2955e99525cc02d9181de67f25515c468856c38d52b5f3b"}, - {file = "ujson-5.10.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:7663960f08cd5a2bb152f5ee3992e1af7690a64c0e26d31ba7b3ff5b2ee66337"}, - {file = "ujson-5.10.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:d8640fb4072d36b08e95a3a380ba65779d356b2fee8696afeb7794cf0902d0a1"}, - {file = "ujson-5.10.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78778a3aa7aafb11e7ddca4e29f46bc5139131037ad628cc10936764282d6753"}, - {file = "ujson-5.10.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0111b27f2d5c820e7f2dbad7d48e3338c824e7ac4d2a12da3dc6061cc39c8e6"}, - {file = "ujson-5.10.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:c66962ca7565605b355a9ed478292da628b8f18c0f2793021ca4425abf8b01e5"}, {file = "ujson-5.10.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ba43cc34cce49cf2d4bc76401a754a81202d8aa926d0e2b79f0ee258cb15d3a4"}, {file = "ujson-5.10.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:ac56eb983edce27e7f51d05bc8dd820586c6e6be1c5216a6809b0c668bb312b8"}, {file = "ujson-5.10.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f44bd4b23a0e723bf8b10628288c2c7c335161d6840013d4d5de20e48551773b"}, @@ -4998,6 +5008,9 @@ name = "wcwidth" version = "0.2.13" summary = "Measures the displayed width of unicode strings in a terminal" groups = ["default"] +dependencies = [ + "backports-functools-lru-cache>=1.2.1; python_version < \"3.2\"", +] files = [ {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"}, {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, @@ -5099,6 +5112,7 @@ groups = ["default", "dev"] dependencies = [ "idna>=2.0", "multidict>=4.0", + "typing-extensions>=3.7.4; python_version < \"3.8\"", ] files = [ {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e"}, diff --git a/backend/sample.celery.env b/backend/sample.celery.env new file mode 100644 index 000000000..832c9c3a3 --- /dev/null +++ b/backend/sample.celery.env @@ -0,0 +1,30 @@ +DJANGO_SETTINGS_MODULE='backend.settings.celery' + +# Default log level +DEFAULT_LOG_LEVEL="INFO" + +# Postgres DB envs +DB_HOST='unstract-db' +DB_USER='unstract_dev' +DB_PASSWORD='unstract_pass' +DB_NAME='unstract_db' +DB_PORT=5432 + +# Redis +REDIS_HOST="unstract-redis" +REDIS_PORT=6379 +REDIS_PASSWORD="" +REDIS_USER=default + +# Protocol buffers generated code. +PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python + +# Enable logging of workflow history. +ENABLE_LOG_HISTORY=True +# Interval in seconds for periodic consumer operations. +LOG_HISTORY_CONSUMER_INTERVAL=30 +# Maximum number of logs to insert in a single batch. +LOGS_BATCH_LIMIT=30 + +# Celery Configuration +CELERY_BROKER_URL="redis://unstract-redis:6379" diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 6e6b9d1c8..b095dac67 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -41,7 +41,7 @@ services: entrypoint: .venv/bin/celery command: "-A backend worker --loglevel=info -Q celery,celery_periodic_logs,celery_log_task_queue --autoscale=8,1" env_file: - - ../backend/.env + - ../backend/.celery.env depends_on: - redis environment: @@ -60,7 +60,7 @@ services: entrypoint: .venv/bin/celery command: "-A backend flower --port=5555 --purge_offline_workers=5" env_file: - - ../backend/.env + - ../backend/.celery.env depends_on: - execution-consumer - redis @@ -83,7 +83,7 @@ services: entrypoint: .venv/bin/celery command: "-A backend beat --scheduler django_celery_beat.schedulers:DatabaseScheduler -l INFO" env_file: - - ../backend/.env + - ../backend/.celery.env - ./essentials.env depends_on: - db diff --git a/prompt-service/pdm.lock b/prompt-service/pdm.lock index 1eef8040f..43320dd4c 100644 --- a/prompt-service/pdm.lock +++ b/prompt-service/pdm.lock @@ -3,19 +3,22 @@ [metadata] groups = ["default", "deploy"] -strategy = ["cross_platform", "inherit_metadata"] -lock_version = "4.4.2" +strategy = ["inherit_metadata"] +lock_version = "4.5.0" content_hash = "sha256:42289642033b05e0330d12a9f44ad5a785bdf2cc5ccace9572996ee05a650bc1" +[[metadata.targets]] +requires_python = ">=3.9,<3.11.1" + [[package]] name = "aiohappyeyeballs" -version = "2.3.5" +version = "2.3.6" requires_python = ">=3.8" summary = "Happy Eyeballs for asyncio" groups = ["default"] files = [ - {file = "aiohappyeyeballs-2.3.5-py3-none-any.whl", hash = "sha256:4d6dea59215537dbc746e93e779caea8178c866856a721c9c660d7a5a7b8be03"}, - {file = "aiohappyeyeballs-2.3.5.tar.gz", hash = "sha256:6fa48b9f1317254f122a07a131a86b71ca6946ca989ce6326fff54a99a920105"}, + {file = "aiohappyeyeballs-2.3.6-py3-none-any.whl", hash = "sha256:15dca2611fa78442f1cb54cf07ffb998573f2b4fbeab45ca8554c045665c896b"}, + {file = "aiohappyeyeballs-2.3.6.tar.gz", hash = "sha256:88211068d2a40e0436033956d7de3926ff36d54776f8b1022d6b21320cadae79"}, ] [[package]] @@ -116,6 +119,9 @@ version = "0.7.0" requires_python = ">=3.8" summary = "Reusable constraint types to use with typing.Annotated" groups = ["default"] +dependencies = [ + "typing-extensions>=4.0.0; python_version < \"3.9\"", +] files = [ {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, @@ -129,6 +135,7 @@ summary = "The official Python library for the anthropic API" groups = ["default"] dependencies = [ "anyio<5,>=3.5.0", + "cached-property; python_version < \"3.8\"", "distro<2,>=1.7.0", "httpx<1,>=0.23.0", "jiter<1,>=0.4.0", @@ -166,6 +173,9 @@ requires_python = ">=3.7" summary = "Timeout context manager for asyncio programs" groups = ["default"] marker = "python_version < \"3.12.0\"" +dependencies = [ + "typing-extensions>=3.6.5; python_version < \"3.8\"", +] files = [ {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, @@ -214,6 +224,9 @@ version = "24.2.0" requires_python = ">=3.7" summary = "Classes Without Boilerplate" groups = ["default"] +dependencies = [ + "importlib-metadata; python_version < \"3.8\"", +] files = [ {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"}, {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"}, @@ -294,23 +307,23 @@ files = [ [[package]] name = "boto3" -version = "1.34.159" +version = "1.34.162" requires_python = ">=3.8" summary = "The AWS SDK for Python" groups = ["default"] dependencies = [ - "botocore<1.35.0,>=1.34.159", + "botocore<1.35.0,>=1.34.162", "jmespath<2.0.0,>=0.7.1", "s3transfer<0.11.0,>=0.10.0", ] files = [ - {file = "boto3-1.34.159-py3-none-any.whl", hash = "sha256:21120d23cc37c0e80dc4f64434bc5664d2a5645dcd9bf8a8fa97ed5c82164ca0"}, - {file = "boto3-1.34.159.tar.gz", hash = "sha256:ffe7bbb88ba81b5d54bc8fa0cfb2f3b7fe63a6cffa0f9207df2ef5c22a1c0587"}, + {file = "boto3-1.34.162-py3-none-any.whl", hash = "sha256:d6f6096bdab35a0c0deff469563b87d184a28df7689790f7fe7be98502b7c590"}, + {file = "boto3-1.34.162.tar.gz", hash = "sha256:873f8f5d2f6f85f1018cbb0535b03cceddc7b655b61f66a0a56995238804f41f"}, ] [[package]] name = "botocore" -version = "1.34.159" +version = "1.34.162" requires_python = ">=3.8" summary = "Low-level, data-driven core of boto 3." groups = ["default"] @@ -321,8 +334,8 @@ dependencies = [ "urllib3<1.27,>=1.25.4; python_version < \"3.10\"", ] files = [ - {file = "botocore-1.34.159-py3-none-any.whl", hash = "sha256:7633062491457419a49f5860c014251ae85689f78266a3ce020c2c8688a76b97"}, - {file = "botocore-1.34.159.tar.gz", hash = "sha256:dc28806eb21e3c8d690c422530dff8b4b242ac033cbe98f160a9d37796c09cb1"}, + {file = "botocore-1.34.162-py3-none-any.whl", hash = "sha256:2d918b02db88d27a75b48275e6fb2506e9adaaddbec1ffa6a8a0898b34e769be"}, + {file = "botocore-1.34.162.tar.gz", hash = "sha256:adc23be4fb99ad31961236342b7cbf3c0bfc62532cd02852196032e8c0d682f3"}, ] [[package]] @@ -461,6 +474,7 @@ summary = "Composable command line interface toolkit" groups = ["default"] dependencies = [ "colorama; platform_system == \"Windows\"", + "importlib-metadata; python_version < \"3.8\"", ] files = [ {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, @@ -784,7 +798,7 @@ files = [ [[package]] name = "google-cloud-aiplatform" -version = "1.61.0" +version = "1.62.0" requires_python = ">=3.8" summary = "Vertex AI API client library" groups = ["default"] @@ -802,8 +816,8 @@ dependencies = [ "shapely<3.0.0dev", ] files = [ - {file = "google-cloud-aiplatform-1.61.0.tar.gz", hash = "sha256:648e3cd7bb75be706d3c31d852a3d4d8a2e616ad4db4cf520ef4430615cf8ad9"}, - {file = "google_cloud_aiplatform-1.61.0-py2.py3-none-any.whl", hash = "sha256:57b36d5fa085e68197e9fc576c43263a7cad320483aa3b166bcd1fdc7e8f49e7"}, + {file = "google-cloud-aiplatform-1.62.0.tar.gz", hash = "sha256:e15d5b2a99e30d4a16f4c51cfb8129962e6da41a9027d2ea696abe0e2f006fe8"}, + {file = "google_cloud_aiplatform-1.62.0-py2.py3-none-any.whl", hash = "sha256:d7738e0fd4494a54ae08a51755a2143d58937cba2db826189771f45566c9ee3c"}, ] [[package]] @@ -835,6 +849,7 @@ groups = ["default"] dependencies = [ "google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.6", "google-auth<3.0dev,>=1.25.0", + "importlib-metadata>1.0.0; python_version < \"3.8\"", ] files = [ {file = "google-cloud-core-2.4.1.tar.gz", hash = "sha256:9b7749272a812bde58fff28868d0c5e2f585b82f37e09a1f6ed2d4d10f134073"}, @@ -917,16 +932,6 @@ files = [ {file = "google_crc32c-1.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7c074fece789b5034b9b1404a1f8208fc2d4c6ce9decdd16e8220c5a793e6f61"}, {file = "google_crc32c-1.5.0-cp39-cp39-win32.whl", hash = "sha256:7f57f14606cd1dd0f0de396e1e53824c371e9544a822648cd76c034d209b559c"}, {file = "google_crc32c-1.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:a2355cba1f4ad8b6988a4ca3feed5bff33f6af2d7f134852cf279c2aebfde541"}, - {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:f314013e7dcd5cf45ab1945d92e713eec788166262ae8deb2cfacd53def27325"}, - {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b747a674c20a67343cb61d43fdd9207ce5da6a99f629c6e2541aa0e89215bcd"}, - {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8f24ed114432de109aa9fd317278518a5af2d31ac2ea6b952b2f7782b43da091"}, - {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8667b48e7a7ef66afba2c81e1094ef526388d35b873966d8a9a447974ed9178"}, - {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:1c7abdac90433b09bad6c43a43af253e688c9cfc1c86d332aed13f9a7c7f65e2"}, - {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:6f998db4e71b645350b9ac28a2167e6632c239963ca9da411523bb439c5c514d"}, - {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c99616c853bb585301df6de07ca2cadad344fd1ada6d62bb30aec05219c45d2"}, - {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ad40e31093a4af319dadf503b2467ccdc8f67c72e4bcba97f8c10cb078207b5"}, - {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd67cf24a553339d5062eff51013780a00d6f97a39ca062781d06b3a73b15462"}, - {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:398af5e3ba9cf768787eef45c803ff9614cc3e22a5b2f7d7ae116df8b11e3314"}, {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:b1f8133c9a275df5613a451e73f36c2aea4fe13c5c8997e22cf355ebd7bd0728"}, {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ba053c5f50430a3fcfd36f75aff9caeba0440b2d076afdb79a318d6ca245f88"}, {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:272d3892a1e1a2dbc39cc5cde96834c236d5327e2122d3aaa19f6614531bb6eb"}, @@ -1168,6 +1173,7 @@ requires_python = ">=3.7" summary = "WSGI HTTP Server for UNIX" groups = ["deploy"] dependencies = [ + "importlib-metadata; python_version < \"3.8\"", "packaging", ] files = [ @@ -1181,6 +1187,9 @@ version = "0.14.0" requires_python = ">=3.7" summary = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" groups = ["default"] +dependencies = [ + "typing-extensions; python_version < \"3.8\"", +] files = [ {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, @@ -1311,6 +1320,7 @@ summary = "Read metadata from Python packages" groups = ["default"] marker = "python_version < \"3.10\"" dependencies = [ + "typing-extensions>=3.6.4; python_version < \"3.8\"", "zipp>=0.5", ] files = [ @@ -1419,7 +1429,9 @@ summary = "An implementation of JSON Schema validation for Python" groups = ["default"] dependencies = [ "attrs>=22.2.0", + "importlib-resources>=1.4.0; python_version < \"3.9\"", "jsonschema-specifications>=2023.03.6", + "pkgutil-resolve-name>=1.3.10; python_version < \"3.9\"", "referencing>=0.28.4", "rpds-py>=0.7.1", ] @@ -1435,6 +1447,7 @@ requires_python = ">=3.8" summary = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" groups = ["default"] dependencies = [ + "importlib-resources>=1.4.0; python_version < \"3.9\"", "referencing>=0.31.0", ] files = [ @@ -1450,6 +1463,7 @@ summary = "Messaging library for Python." groups = ["default"] dependencies = [ "amqp<6.0.0,>=5.1.1", + "backports-zoneinfo[tzdata]>=0.2.1; python_version < \"3.9\"", "typing-extensions; python_version < \"3.10\"", "vine", ] @@ -1641,7 +1655,7 @@ files = [ [[package]] name = "llama-index-legacy" -version = "0.9.48.post1" +version = "0.9.48.post2" requires_python = "<4.0,>=3.8.1" summary = "Interface between LLMs and your data" groups = ["default"] @@ -1655,7 +1669,7 @@ dependencies = [ "httpx", "nest-asyncio<2.0.0,>=1.5.8", "networkx>=3.0", - "nltk>=3.8.2", + "nltk>=3.8.1", "numpy", "openai>=1.1.0", "pandas", @@ -1666,8 +1680,8 @@ dependencies = [ "typing-inspect>=0.8.0", ] files = [ - {file = "llama_index_legacy-0.9.48.post1-py3-none-any.whl", hash = "sha256:583296162385010ebf92d2a612dd0a504575c04dc1638323bb455b1521aabe57"}, - {file = "llama_index_legacy-0.9.48.post1.tar.gz", hash = "sha256:e8b1603929433fd0cf3287ed700714078534dd202c97bcdbcc83ec3741bb0868"}, + {file = "llama_index_legacy-0.9.48.post2-py3-none-any.whl", hash = "sha256:2581af680a4e577d4f0accd76e8286c5f1054f28a2fb0e8e5758f09ce5da0176"}, + {file = "llama_index_legacy-0.9.48.post2.tar.gz", hash = "sha256:a4c1f10b4d19d005674195c449f4e859022c65c816dcba1a619ef5df922aa212"}, ] [[package]] @@ -2050,7 +2064,7 @@ files = [ [[package]] name = "milvus-lite" -version = "2.4.8" +version = "2.4.9" requires_python = ">=3.7" summary = "A lightweight version of Milvus wrapped with Python." groups = ["default"] @@ -2059,10 +2073,10 @@ dependencies = [ "tqdm", ] files = [ - {file = "milvus_lite-2.4.8-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:b7e90b34b214884cd44cdc112ab243d4cb197b775498355e2437b6cafea025fe"}, - {file = "milvus_lite-2.4.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:519dfc62709d8f642d98a1c5b1dcde7080d107e6e312d677fef5a3412a40ac08"}, - {file = "milvus_lite-2.4.8-py3-none-manylinux2014_aarch64.whl", hash = "sha256:b21f36d24cbb0e920b4faad607019bb28c1b2c88b4d04680ac8c7697a4ae8a4d"}, - {file = "milvus_lite-2.4.8-py3-none-manylinux2014_x86_64.whl", hash = "sha256:08332a2b9abfe7c4e1d7926068937e46f8fb81f2707928b7bc02c9dc99cebe41"}, + {file = "milvus_lite-2.4.9-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:d3e617b3d68c09ad656d54bc3d8cc4ef6ef56c54015e1563d4fe4bcec6b7c90a"}, + {file = "milvus_lite-2.4.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:6e7029282d6829b277ebb92f64e2370be72b938e34770e1eb649346bda5d1d7f"}, + {file = "milvus_lite-2.4.9-py3-none-manylinux2014_aarch64.whl", hash = "sha256:9b8e991e4e433596f6a399a165c1a506f823ec9133332e03d7f8a114bff4550d"}, + {file = "milvus_lite-2.4.9-py3-none-manylinux2014_x86_64.whl", hash = "sha256:7f53e674602101cfbcf0a4a59d19eaa139dfd5580639f3040ad73d901f24fc0b"}, ] [[package]] @@ -2203,8 +2217,8 @@ files = [ [[package]] name = "nltk" -version = "3.8.2" -requires_python = ">=3.8" +version = "3.8.1" +requires_python = ">=3.7" summary = "Natural Language Toolkit" groups = ["default"] dependencies = [ @@ -2214,8 +2228,8 @@ dependencies = [ "tqdm", ] files = [ - {file = "nltk-3.8.2-py3-none-any.whl", hash = "sha256:bae044ae22ebe0b694a87c0012233373209f27d5c76d3572599c842740a62fe0"}, - {file = "nltk-3.8.2.tar.gz", hash = "sha256:9c051aa981c6745894906d5c3aad27417f3d1c10d91eefca50382fc922966f31"}, + {file = "nltk-3.8.1-py3-none-any.whl", hash = "sha256:fd5c9109f976fa86bcadba8f91e47f5e9293bd034474752e92a520f81c93dda5"}, + {file = "nltk-3.8.1.zip", hash = "sha256:1834da3d0682cba4f2cede2f9aad6b0fafb6461ba451db0efb6f9c39798d64d3"}, ] [[package]] @@ -2271,12 +2285,13 @@ files = [ [[package]] name = "openai" -version = "1.40.6" +version = "1.40.8" requires_python = ">=3.7.1" summary = "The official Python library for the openai API" groups = ["default"] dependencies = [ "anyio<5,>=3.5.0", + "cached-property; python_version < \"3.8\"", "distro<2,>=1.7.0", "httpx<1,>=0.23.0", "jiter<1,>=0.4.0", @@ -2286,8 +2301,8 @@ dependencies = [ "typing-extensions<5,>=4.11", ] files = [ - {file = "openai-1.40.6-py3-none-any.whl", hash = "sha256:b36372124a779381a420a34dd96f762baa748b6bdfaf83a6b9f2745f72ccc1c5"}, - {file = "openai-1.40.6.tar.gz", hash = "sha256:2239232bcb7f4bd4ce8e02544b5769618582411cf399816d96686d1b6c1e5c8d"}, + {file = "openai-1.40.8-py3-none-any.whl", hash = "sha256:3ed4ddad48e0dde059c9b4d3dc240e47781beca2811e52ba449ddc4a471a2fd4"}, + {file = "openai-1.40.8.tar.gz", hash = "sha256:e225f830b946378e214c5b2cfa8df28ba2aeb7e9d44f738cb2a926fd971f5bc0"}, ] [[package]] @@ -2350,6 +2365,7 @@ groups = ["default"] dependencies = [ "numpy>=1.22.4; python_version < \"3.11\"", "numpy>=1.23.2; python_version == \"3.11\"", + "numpy>=1.26.0; python_version >= \"3.12\"", "python-dateutil>=2.8.2", "pytz>=2020.1", "tzdata>=2022.7", @@ -2388,6 +2404,8 @@ groups = ["default"] dependencies = [ "charset-normalizer>=2.0.0", "cryptography>=36.0.0", + "importlib-metadata; python_version < \"3.8\"", + "typing-extensions; python_version < \"3.8\"", ] files = [ {file = "pdfminer.six-20231228-py3-none-any.whl", hash = "sha256:e8d3c3310e6fbc1fe414090123ab01351634b4ecb021232206c4c9a8ca3e3b8f"}, @@ -2500,6 +2518,7 @@ dependencies = [ "tqdm>=4.64.1", "typing-extensions>=3.7.4", "urllib3>=1.26.0; python_version >= \"3.8\" and python_version < \"3.12\"", + "urllib3>=1.26.5; python_version ~= \"3.12\"", ] files = [ {file = "pinecone_client-3.2.2-py3-none-any.whl", hash = "sha256:7e492fdda23c73726bc0cb94c689bb950d06fb94e82b701a0c610c2e830db327"}, @@ -2678,6 +2697,7 @@ groups = ["default"] dependencies = [ "annotated-types>=0.4.0", "pydantic-core==2.20.1", + "typing-extensions>=4.12.2; python_version >= \"3.13\"", "typing-extensions>=4.6.1; python_version < \"3.13\"", ] files = [ @@ -2787,8 +2807,10 @@ dependencies = [ "environs<=9.5.0", "grpcio<=1.63.0,>=1.49.1", "milvus-lite<2.5.0,>=2.4.0; sys_platform != \"win32\"", + "numpy<1.25.0; python_version <= \"3.8\"", "pandas>=1.2.4", "protobuf>=3.20.0", + "setuptools<70.1; python_version <= \"3.8\"", "setuptools>69", "ujson>=2.0.0", ] @@ -2804,6 +2826,7 @@ requires_python = ">=3.6" summary = "A pure-python PDF library capable of splitting, merging, cropping, and transforming PDF files" groups = ["default"] dependencies = [ + "dataclasses; python_version < \"3.7\"", "typing-extensions>=4.0; python_version < \"3.11\"", ] files = [ @@ -2942,7 +2965,9 @@ dependencies = [ "grpcio-tools>=1.41.0", "grpcio>=1.41.0", "httpx[http2]>=0.20.0", + "numpy<1.21; python_version < \"3.8\"", "numpy>=1.21; python_version >= \"3.8\" and python_version < \"3.12\"", + "numpy>=1.26; python_version >= \"3.12\"", "portalocker<3.0.0,>=2.7.0", "pydantic>=1.10.8", "urllib3<3,>=1.26.14", @@ -2960,6 +2985,8 @@ summary = "Python client for Redis database and key-value store" groups = ["default"] dependencies = [ "async-timeout>=4.0.3; python_full_version < \"3.11.3\"", + "importlib-metadata>=1.0; python_version < \"3.8\"", + "typing-extensions; python_version < \"3.8\"", ] files = [ {file = "redis-5.0.8-py3-none-any.whl", hash = "sha256:56134ee08ea909106090934adc36f65c9bcbbaecea5b21ba704ba6fb561f8eb4"}, @@ -3206,19 +3233,6 @@ files = [ {file = "safetensors-0.4.4-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e5c9d86d9b13b18aafa88303e2cd21e677f5da2a14c828d2c460fe513af2e9a5"}, {file = "safetensors-0.4.4-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:43251d7f29a59120a26f5a0d9583b9e112999e500afabcfdcb91606d3c5c89e3"}, {file = "safetensors-0.4.4-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:2c42e9b277513b81cf507e6121c7b432b3235f980cac04f39f435b7902857f91"}, - {file = "safetensors-0.4.4-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3daacc9a4e3f428a84dd56bf31f20b768eb0b204af891ed68e1f06db9edf546f"}, - {file = "safetensors-0.4.4-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:218bbb9b883596715fc9997bb42470bf9f21bb832c3b34c2bf744d6fa8f2bbba"}, - {file = "safetensors-0.4.4-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7bd5efc26b39f7fc82d4ab1d86a7f0644c8e34f3699c33f85bfa9a717a030e1b"}, - {file = "safetensors-0.4.4-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:56ad9776b65d8743f86698a1973292c966cf3abff627efc44ed60e66cc538ddd"}, - {file = "safetensors-0.4.4-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:30f23e6253c5f43a809dea02dc28a9f5fa747735dc819f10c073fe1b605e97d4"}, - {file = "safetensors-0.4.4-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:5512078d00263de6cb04e9d26c9ae17611098f52357fea856213e38dc462f81f"}, - {file = "safetensors-0.4.4-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b96c3d9266439d17f35fc2173111d93afc1162f168e95aed122c1ca517b1f8f1"}, - {file = "safetensors-0.4.4-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:08d464aa72a9a13826946b4fb9094bb4b16554bbea2e069e20bd903289b6ced9"}, - {file = "safetensors-0.4.4-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:210160816d5a36cf41f48f38473b6f70d7bcb4b0527bedf0889cc0b4c3bb07db"}, - {file = "safetensors-0.4.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb276a53717f2bcfb6df0bcf284d8a12069002508d4c1ca715799226024ccd45"}, - {file = "safetensors-0.4.4-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a2c28c6487f17d8db0089e8b2cdc13de859366b94cc6cdc50e1b0a4147b56551"}, - {file = "safetensors-0.4.4-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:7915f0c60e4e6e65d90f136d85dd3b429ae9191c36b380e626064694563dbd9f"}, - {file = "safetensors-0.4.4-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:00eea99ae422fbfa0b46065acbc58b46bfafadfcec179d4b4a32d5c45006af6c"}, {file = "safetensors-0.4.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:bb1ed4fcb0b3c2f3ea2c5767434622fe5d660e5752f21ac2e8d737b1e5e480bb"}, {file = "safetensors-0.4.4-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:73fc9a0a4343188bdb421783e600bfaf81d0793cd4cce6bafb3c2ed567a74cd5"}, {file = "safetensors-0.4.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c37e6b714200824c73ca6eaf007382de76f39466a46e97558b8dc4cf643cfbf"}, @@ -3231,13 +3245,13 @@ files = [ [[package]] name = "setuptools" -version = "72.1.0" +version = "72.2.0" requires_python = ">=3.8" summary = "Easily download, build, install, upgrade, and uninstall Python packages" groups = ["default"] files = [ - {file = "setuptools-72.1.0-py3-none-any.whl", hash = "sha256:5a03e1860cf56bb6ef48ce186b0e557fdba433237481a9a625176c2831be15d1"}, - {file = "setuptools-72.1.0.tar.gz", hash = "sha256:8d243eff56d095e5817f796ede6ae32941278f542e0f941867cc05ae52b162ec"}, + {file = "setuptools-72.2.0-py3-none-any.whl", hash = "sha256:f11dd94b7bae3a156a95ec151f24e4637fb4fa19c878e4d191bfb8b2d82728c4"}, + {file = "setuptools-72.2.0.tar.gz", hash = "sha256:80aacbf633704e9c8bfa1d99fa5dd4dc59573efcf9e4042c13d3bcef91ac2ef9"}, ] [[package]] @@ -3304,13 +3318,13 @@ files = [ [[package]] name = "soupsieve" -version = "2.5" +version = "2.6" requires_python = ">=3.8" summary = "A modern CSS selector implementation for Beautiful Soup." groups = ["default"] files = [ - {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"}, - {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"}, + {file = "soupsieve-2.6-py3-none-any.whl", hash = "sha256:e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9"}, + {file = "soupsieve-2.6.tar.gz", hash = "sha256:e2e68417777af359ec65daac1057404a3c8a5455bb8abc36f1a9866ab1a51abb"}, ] [[package]] @@ -3321,6 +3335,7 @@ summary = "Database Abstraction Library" groups = ["default"] dependencies = [ "greenlet!=0.4.17; (platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\") and python_version < \"3.13\"", + "importlib-metadata; python_version < \"3.8\"", "typing-extensions>=4.6.0", ] files = [ @@ -3501,19 +3516,6 @@ files = [ {file = "tokenizers-0.15.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67a0fe1e49e60c664915e9fb6b0cb19bac082ab1f309188230e4b2920230edb3"}, {file = "tokenizers-0.15.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:4e022fe65e99230b8fd89ebdfea138c24421f91c1a4f4781a8f5016fd5cdfb4d"}, {file = "tokenizers-0.15.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d857be2df69763362ac699f8b251a8cd3fac9d21893de129bc788f8baaef2693"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:708bb3e4283177236309e698da5fcd0879ce8fd37457d7c266d16b550bcbbd18"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:64c35e09e9899b72a76e762f9854e8750213f67567787d45f37ce06daf57ca78"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1257f4394be0d3b00de8c9e840ca5601d0a4a8438361ce9c2b05c7d25f6057b"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02272fe48280e0293a04245ca5d919b2c94a48b408b55e858feae9618138aeda"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:dc3ad9ebc76eabe8b1d7c04d38be884b8f9d60c0cdc09b0aa4e3bcf746de0388"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:32e16bdeffa7c4f46bf2152172ca511808b952701d13e7c18833c0b73cb5c23f"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:fb16ba563d59003028b678d2361a27f7e4ae0ab29c7a80690efa20d829c81fdb"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:2277c36d2d6cdb7876c274547921a42425b6810d38354327dd65a8009acf870c"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1cf75d32e8d250781940d07f7eece253f2fe9ecdb1dc7ba6e3833fa17b82fcbc"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1b3b31884dc8e9b21508bb76da80ebf7308fdb947a17affce815665d5c4d028"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b10122d8d8e30afb43bb1fe21a3619f62c3e2574bff2699cf8af8b0b6c5dc4a3"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d88b96ff0fe8e91f6ef01ba50b0d71db5017fa4e3b1d99681cec89a85faf7bf7"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:37aaec5a52e959892870a7c47cef80c53797c0db9149d458460f4f31e2fb250e"}, {file = "tokenizers-0.15.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e2ea752f2b0fe96eb6e2f3adbbf4d72aaa1272079b0dfa1145507bd6a5d537e6"}, {file = "tokenizers-0.15.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:4b19a808d8799fda23504a5cd31d2f58e6f52f140380082b352f877017d6342b"}, {file = "tokenizers-0.15.2-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:64c86e5e068ac8b19204419ed8ca90f9d25db20578f5881e337d203b314f4104"}, @@ -3580,6 +3582,7 @@ groups = ["default"] dependencies = [ "mypy-extensions>=0.3.0", "typing-extensions>=3.7.4", + "typing>=3.7.4; python_version < \"3.5\"", ] files = [ {file = "typing_inspect-0.9.0-py3-none-any.whl", hash = "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f"}, @@ -3640,11 +3643,6 @@ files = [ {file = "ujson-5.10.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fbd8fd427f57a03cff3ad6574b5e299131585d9727c8c366da4624a9069ed746"}, {file = "ujson-5.10.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:beeaf1c48e32f07d8820c705ff8e645f8afa690cca1544adba4ebfa067efdc88"}, {file = "ujson-5.10.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:baed37ea46d756aca2955e99525cc02d9181de67f25515c468856c38d52b5f3b"}, - {file = "ujson-5.10.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:7663960f08cd5a2bb152f5ee3992e1af7690a64c0e26d31ba7b3ff5b2ee66337"}, - {file = "ujson-5.10.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:d8640fb4072d36b08e95a3a380ba65779d356b2fee8696afeb7794cf0902d0a1"}, - {file = "ujson-5.10.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78778a3aa7aafb11e7ddca4e29f46bc5139131037ad628cc10936764282d6753"}, - {file = "ujson-5.10.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0111b27f2d5c820e7f2dbad7d48e3338c824e7ac4d2a12da3dc6061cc39c8e6"}, - {file = "ujson-5.10.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:c66962ca7565605b355a9ed478292da628b8f18c0f2793021ca4425abf8b01e5"}, {file = "ujson-5.10.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ba43cc34cce49cf2d4bc76401a754a81202d8aa926d0e2b79f0ee258cb15d3a4"}, {file = "ujson-5.10.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:ac56eb983edce27e7f51d05bc8dd820586c6e6be1c5216a6809b0c668bb312b8"}, {file = "ujson-5.10.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f44bd4b23a0e723bf8b10628288c2c7c335161d6840013d4d5de20e48551773b"}, @@ -3846,6 +3844,7 @@ groups = ["default"] dependencies = [ "idna>=2.0", "multidict>=4.0", + "typing-extensions>=3.7.4; python_version < \"3.8\"", ] files = [ {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e"}, diff --git a/run-platform.sh b/run-platform.sh index fea90dbc8..67f3e18a3 100755 --- a/run-platform.sh +++ b/run-platform.sh @@ -141,42 +141,45 @@ do_git_pull() { if [[ "$opt_version" == "latest" ]]; then branch=`git describe --tags --abbrev=0` - elif [[ "$opt_version" == "main" ]]; then - branch="main" - opt_build_local=true - echo -e "Choosing ""$blue_text""local build""$default_text"" of Docker images from ""$blue_text""main""$default_text"" branch." - elif [ -z $(git tag -l "$opt_version") ]; then - echo -e "$red_text""Version not found.""$default_text" - if [[ ! $opt_version == v* ]]; then - echo -e "$red_text""Version must be provided with a 'v' prefix (e.g. v0.47.0).""$default_text" + elif [[ $opt_version == v* ]]; then + if [ -z $(git tag -l "$opt_version") ]; then + echo -e "$red_text""Version not found.""$default_text" + exit 1 fi - exit 1 - else branch="$opt_version" + elif [[ "$opt_version" == "current" ]]; then + branch=`git branch --show-current` + opt_build_local=true + echo -e "Opting ""$blue_text""local build""$default_text"" of Docker images from ""$blue_text""$branch""$default_text"" branch." + else + echo -e "$red_text""Version must be one of {latest, vX.Y.Z (e.g. v0.47.0), current}.""$default_text" + exit 1 fi - echo -e "Performing ""$blue_text""git checkout""$default_text"" to ""$blue_text""$branch""$default_text""." - git checkout --quiet $branch + if [[ "$opt_version" == "latest" ]] || [[ $opt_version == v* ]]; then + echo -e "Performing ""$blue_text""git checkout""$default_text"" to ""$blue_text""$branch""$default_text""." + git checkout --quiet $branch - echo -e "Performing ""$blue_text""git pull""$default_text"" on ""$blue_text""$branch""$default_text""." - git pull --quiet $(git remote) $branch + echo -e "Performing ""$blue_text""git pull""$default_text"" on ""$blue_text""$branch""$default_text""." + git pull --quiet $(git remote) $branch + fi } -copy_or_merge_envs() { - - local src_file="$1" - local dest_file="$2" - local displayed_reason="$3" +_copy_or_merge_envs() { + local src_file_path="$1" + local dest_file_path="$2" + local service="$3" - if [ ! -e "$dest_file" ]; then - cp "$src_file" "$dest_file" - echo -e "Created env for ""$blue_text""$displayed_reason""$default_text"" at ""$blue_text""$dest_file""$default_text""." + if [ -e "$src_file_path" ] && [ ! -e "$dest_file_path" ]; then + first_setup=true + cp "$src_file_path" "$dest_file_path" + echo -e "Created env for ""$blue_text""$service""$default_text"" at ""$blue_text""$dest_file_path""$default_text""." elif [ "$opt_only_env" = true ] || [ "$opt_update" = true ]; then - python3 $script_dir/docker/scripts/merge_env.py "$src_file" "$dest_file" + python3 $script_dir/docker/scripts/merge_env.py "$src_file_path" "$dest_file_path" if [ $? -ne 0 ]; then exit 1 fi - echo -e "Merged env for ""$blue_text""$displayed_reason""$default_text"" at ""$blue_text""$dest_file""$default_text""." + echo -e "Merged env for ""$blue_text""$service""$default_text"" at ""$blue_text""$dest_file_path""$default_text""." fi } @@ -189,10 +192,9 @@ setup_env() { sample_env_path="$script_dir/$service/sample.env" env_path="$script_dir/$service/.env" - if [ -e "$sample_env_path" ] && [ ! -e "$env_path" ]; then - first_setup=true - cp "$sample_env_path" "$env_path" + _copy_or_merge_envs $sample_env_path $env_path $service + if [ "$first_setup" = true ]; then # Add encryption secret for backend and platform-service. if [[ "$service" == "backend" || "$service" == "platform-service" ]]; then echo -e "$blue_text""Adding encryption secret to $service""$default_text" @@ -217,19 +219,11 @@ setup_env() { # sed -i "s/SYSTEM_ADMIN_PASSWORD.*/SYSTEM_ADMIN_PASSWORD=\"$DEFAULT_AUTH_KEY\"/" $env_path fi fi - echo -e "Created env for ""$blue_text""$service""$default_text" at ""$blue_text""$env_path""$default_text"." - elif [ "$opt_only_env" = true ] || [ "$opt_update" = true ]; then - python3 $script_dir/docker/scripts/merge_env.py $sample_env_path $env_path - if [ $? -ne 0 ]; then - exit 1 - fi - echo -e "Merged env for ""$blue_text""$service""$default_text" at ""$blue_text""$env_path""$default_text"." fi done - copy_or_merge_envs "$script_dir/docker/sample.essentials.env" "$script_dir/docker/essentials.env" "essential services" - copy_or_merge_envs "$script_dir/docker/sample.env" "$script_dir/docker/.env" "docker compose" - + _copy_or_merge_envs "$script_dir/docker/sample.essentials.env" "$script_dir/docker/essentials.env" "essential services" + _copy_or_merge_envs "$script_dir/docker/sample.env" "$script_dir/docker/.env" "docker compose" if [ "$opt_only_env" = true ]; then echo -e "$green_text""Done.""$default_text" && exit 0 @@ -240,7 +234,7 @@ build_services() { pushd ${script_dir}/docker 1>/dev/null if [ "$opt_build_local" = true ]; then - echo -e "$blue_text""Building""$default_text"" docker images ""$blue_text""$opt_version""$default_text"" locally." + echo -e "$blue_text""Building""$default_text"" docker images tag ""$blue_text""$opt_version""$default_text"" locally." VERSION=$opt_version $docker_compose_cmd -f $script_dir/docker/docker-compose.build.yaml build || { echo -e "$red_text""Failed to build docker images.""$default_text" exit 1 From d285c4d13fd4468e3d8679d97cbf46d5580b45ee Mon Sep 17 00:00:00 2001 From: Hari John Kuriakose Date: Fri, 16 Aug 2024 21:21:50 +0530 Subject: [PATCH 3/6] Update gitignore for env files --- backend/.celery.env | 30 ------------------------------ 1 file changed, 30 deletions(-) delete mode 100644 backend/.celery.env diff --git a/backend/.celery.env b/backend/.celery.env deleted file mode 100644 index 832c9c3a3..000000000 --- a/backend/.celery.env +++ /dev/null @@ -1,30 +0,0 @@ -DJANGO_SETTINGS_MODULE='backend.settings.celery' - -# Default log level -DEFAULT_LOG_LEVEL="INFO" - -# Postgres DB envs -DB_HOST='unstract-db' -DB_USER='unstract_dev' -DB_PASSWORD='unstract_pass' -DB_NAME='unstract_db' -DB_PORT=5432 - -# Redis -REDIS_HOST="unstract-redis" -REDIS_PORT=6379 -REDIS_PASSWORD="" -REDIS_USER=default - -# Protocol buffers generated code. -PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python - -# Enable logging of workflow history. -ENABLE_LOG_HISTORY=True -# Interval in seconds for periodic consumer operations. -LOG_HISTORY_CONSUMER_INTERVAL=30 -# Maximum number of logs to insert in a single batch. -LOGS_BATCH_LIMIT=30 - -# Celery Configuration -CELERY_BROKER_URL="redis://unstract-redis:6379" From 267e122c3aa780d654c98b90239a70fe7b436b46 Mon Sep 17 00:00:00 2001 From: Hari John Kuriakose Date: Fri, 16 Aug 2024 21:26:47 +0530 Subject: [PATCH 4/6] Update gitignore for env files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3a88d7c94..760f6184d 100644 --- a/.gitignore +++ b/.gitignore @@ -135,6 +135,7 @@ celerybeat.pid # Environments test*.env .env +.*.env .env.export .venv* env/ From 3283054257abeab925bc4e7631d972f6d15b165f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 8 Nov 2024 10:27:38 +0000 Subject: [PATCH 5/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- backend/backend/settings/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py index a86bcab28..67d1cb7c3 100644 --- a/backend/backend/settings/base.py +++ b/backend/backend/settings/base.py @@ -12,7 +12,6 @@ import os from pathlib import Path from typing import Optional -from urllib.parse import quote_plus from dotenv import find_dotenv, load_dotenv from utils.common_utils import CommonUtils From 036637b0715bbfce6d74bc803ccd3d56f5748b02 Mon Sep 17 00:00:00 2001 From: Hari John Kuriakose Date: Fri, 8 Nov 2024 16:24:05 +0530 Subject: [PATCH 6/6] Update run-platform.sh Signed-off-by: Hari John Kuriakose --- run-platform.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/run-platform.sh b/run-platform.sh index 129093b28..510ff9c9a 100755 --- a/run-platform.sh +++ b/run-platform.sh @@ -146,7 +146,7 @@ do_git_pull() { elif [[ "$opt_version" == "current" ]]; then target_branch=`git branch --show-current` opt_build_local=true - echo -e "Opting ""$blue_text""local build""$default_text"" of Docker images from ""$blue_text""$branch""$default_text"" branch." + echo -e "Opting ""$blue_text""local build""$default_text"" of Docker images from ""$blue_text""$target_branch""$default_text"" branch." elif [ -z $(git tag -l "$opt_version") ]; then echo -e "$red_text""Version not found.""$default_text" version_regex="^v([0-9]+)\.([0-9]+)\.([0-9]+)(-[a-zA-Z0-9]+(\.[0-9]+)?)?$" @@ -165,7 +165,7 @@ do_git_pull() { git pull --quiet $(git remote) $target_branch } -_copy_or_merge_envs() { +_copy_or_merge_env() { local src_file_path="$1" local dest_file_path="$2" local service="$3" @@ -192,7 +192,7 @@ setup_env() { sample_env_path="$script_dir/$service/sample.env" env_path="$script_dir/$service/.env" - _copy_or_merge_envs $sample_env_path $env_path $service + _copy_or_merge_env $sample_env_path $env_path $service if [ "$first_setup" = true ]; then # Add encryption secret for backend and platform-service. @@ -222,8 +222,8 @@ setup_env() { fi done - _copy_or_merge_envs "$script_dir/docker/sample.essentials.env" "$script_dir/docker/essentials.env" "essential services" - _copy_or_merge_envs "$script_dir/docker/sample.env" "$script_dir/docker/.env" "docker compose" + _copy_or_merge_env "$script_dir/docker/sample.essentials.env" "$script_dir/docker/essentials.env" "essential services" + _copy_or_merge_env "$script_dir/docker/sample.env" "$script_dir/docker/.env" "docker compose" if [ "$opt_only_env" = true ]; then echo -e "$green_text""Done.""$default_text" && exit 0