diff --git a/ivy/transpiler/caching/assets/ivy_cache_assets.py b/ivy/transpiler/caching/assets/ivy_cache_assets.py deleted file mode 100644 index 84387cfb485c7..0000000000000 --- a/ivy/transpiler/caching/assets/ivy_cache_assets.py +++ /dev/null @@ -1,110 +0,0 @@ -ALL_IVY_FUNCTIONS = { - "ivy": [ - "pad", - "conv_general_dilated", - "batch_norm", - "layer_norm", - "linear", - "dropout", - "gelu", - "relu", - "softmax", - "sigmoid", - "leaky_relu", - "interpolate", - "max_pool2d", - "reshape", - "swapaxes", - "flatten", - "expand_dims", - "permute_dims", - "stack", - "concat", - "meshgrid", - "sign", - "erfinv", - "add", - "pow", - "multiply", - "divide", - "floor_divide", - "exp", - "log", - "log2", - "abs", - "sign", - "minimum", - "maximum", - "roll", - "clip", - "split", - "nonzero", - "argwhere", - "where", - "array", - "asarray", - "is_array", - "empty", - "to_scalar", - "to_numpy", - "ones", - "full_like", - "zeros", - "zeros_like", - "random_uniform", - "linspace", - "bernoulli", - "arange", - "inplace_update", - "gather_nd", - "scatter_nd", - "any", - "all", - "equal", - "dtype", - "default_dtype", - "astype", - "as_native_dtype", - "as_ivy_dtype", - "is_int_dtype", - "dtype_bits", - "default_uint_dtype", - "default_float_dtype", - "promote_types", - "is_uint_dtype", - "default_int_dtype", - "is_bool_dtype", - "promote_types_of_inputs", - "default_complex_dtype", - "is_float_dtype", - "broadcast_arrays", - "is_native_dtype", - "infer_default_dtype", - "is_complex_dtype", - "default_device", - "dev", - "to_device", - "as_ivy_dev", - "as_native_dev", - "map", - "nested_map", - "nested_argwhere", - "index_nest", - "utils.assertions.check_all", - "utils.assertions.check_all_or_any_fn", - "utils.assertions.check_any", - "utils.assertions.check_elem_in_list", - "utils.assertions.check_exists", - "utils.assertions.check_equal", - "utils.assertions.check_shapes_broadcastable", - "handle_array_like_without_promotion", - "infer_dtype", - "handle_partial_mixed_function", - "stop_gradient", - ], - "ivy.functional.ivy.general": ["get_item", "set_item"], - "ivy.functional.ivy.creation": [ - "_asarray_infer_dtype", - "_asarray_to_native_arrays_and_back", - ], -} diff --git a/ivy/transpiler/caching/assets/torch_cache_assets.py b/ivy/transpiler/caching/assets/torch_cache_assets.py deleted file mode 100644 index c6cf0730329a7..0000000000000 --- a/ivy/transpiler/caching/assets/torch_cache_assets.py +++ /dev/null @@ -1,574 +0,0 @@ -ALL_TORCH_FRONTEND_FUNCTIONS = { - "torch": [ - "abs", - "absolute", - "acos", - "acosh", - "add", - "addbmm", - "addcdiv", - "addcmul", - "addmm", - "addmv", - "addr", - # "adjoint", - "all", - "allclose", - "amax", - "amin", - # "aminmax", - "angle", - "any", - "arange", - # "arccos", - # "arccosh", - # "arcsin", - # "arcsinh", - # "arctan", - "arctan2", - "arctanh", - "argmax", - "argmin", - # "argsort", - # "argwhere", - "as_strided", - "as_tensor", - # "asarray", - "asin", - # "asinh", - "atan", - "atan2", - "atanh", - # "atleast_1d", - # "atleast_2d", - # "atleast_3d", - "baddbmm", - # "bartlett_window", - "bernoulli", - "bincount", - "bitwise_and", - # "bitwise_left_shift", - "bitwise_not", - "bitwise_or", - # "bitwise_right_shift", - "bitwise_xor", - # "blackman_window", - # "block_diag", - # "bmm", - # "broadcast_shapes", - # "broadcast_tensors", - # "broadcast_to", - # "can_cast", - # "cartesian_prod", - "cat", - "cdist", - "ceil", - # "chain_matmul", - "cholesky", - "chunk", - "clamp", - "clip", - "clone", - "column_stack", - "complex", - "concat", - "conj", - # "conj_physical", - # "copysign", - # "corrcoef", - "cos", - "cosh", - # "count_nonzero", - # "cov", - "cross", - # "cummax", - # "cumprod", - "cumsum", - # "deg2rad", - "det", - "diag", - # "diag_embed", - "diagflat", - "diagonal", - # "diagonal_scatter", - # "diff", - # "dist", - "div", - "divide", - "dot", - "einsum", - "empty", - # "empty_like", - # "empty_strided", - "eq", - "equal", - "erf", - "erfc", - # "erfinv", - "exp", - "exp2", - "expm1", - "eye", - "finfo", - # "fix", - "flatten", - "flip", - "floor", - "floor_divide", - "fmod", - "frac", - "full", - "full_like", - "gather", - # "gcd", - "ge", - "ger", - # "get_default_dtype", - # "gradient", - "greater", - "greater_equal", - "gt", - "index_add", - "index_copy", - # "index_reduce", - "index_select", - "inverse", - # "is_complex", - # "is_floating_point", - # "is_nonzero", - # "is_tensor", - "isclose", - "isfinite", - "isinf", - "isnan", - # "isneginf", - # "isposinf", - # "isreal", - # "kaiser_window", - # "kron", - # "kthvalue", - # "lcm", - # "ldexp", - "le", - "lerp", - "less", - "less_equal", - "lgamma", - "linspace", - "log", - "log10", - "log1p", - "log2", - "logaddexp", - # "logaddexp2", - "logcumsumexp", - "logdet", - # "logical_and", - # "logical_not", - # "logical_or", - # "logical_xor", - # "logit", - # logspace, - "logsumexp", - # "lstm", - "lt", - # "lu_solve", - # "manual_seed", - "masked_fill", - # "masked_select", - "matmul", - # "matrix_power", - # "matrix_rank", - "max", - "maximum", - "mean", - "median", - "meshgrid", - "min", - "minimum", - "mm", - # "moveaxis", - # "movedim", - # "msort", - "mul", - "multinomial", - "multiply", - "mv", - # "mvlgamma", - "nan_to_num", - # "nanmean", - # "nanmedian", - # "nansum", - "narrow", - "ne", - # "negative", - # "nextafter", - "nonzero", - "norm", - "normal", - # "not_equal", - "numel", - "ones", - "ones_like", - "outer", - "permute", - "pinverse", - # "poisson", - # "polar", - # "positive", - "pow", - "prod", - "qr", - # "quantile", - # "rad2deg", - "rand", - "rand_like", - "randint", - "randint_like", - "randn", - "randn_like", - "randperm", - "range", - # "ravel", - # "real", - "reciprocal", - "relu", - "remainder", - # "renorm", - # "repeat_interleave", - "reshape", - "result_type", - "roll", - "rot90", - "round", - "rsqrt", - "scatter", - "scatter_add", - # "scatter_reduce", - # "searchsorted", - # "seed", - "select", - # "set_default_dtype", - # "set_rng_state", - # "sgn", - "sigmoid", - "sign", - "sin", - "sinh", - # "slogdet", - "softmax", - "sort", - "split", - "sqrt", - "square", - "squeeze", - "stack", - "std", - "std_mean", - "sub", - "subtract", - "sum", - # "svd", - "swapaxes", - # "swapdims", - "t", - # "take", - # "take_along_dim", - "tan", - "tanh", - "tensor", - # "tensor_split", - "tensordot", - "tile", - "topk", - "transpose", - "tril", - "tril_indices", - "triu", - "triu_indices", - "true_divide", - "trunc", - "unbind", - "unflatten", - "unique", - "unique_consecutive", - "unsqueeze", - # "vander", - "var", - "var_mean", - # "vdot", - # "view_as_complex", - # "view_as_real", - # "vmap", - # "vsplit", - # "vstack", - "where", - # "xlogy", - "zeros", - "zeros_like", - ], - "torch.nn.functional": [ - "adaptive_avg_pool1d", - "adaptive_avg_pool2d", - "adaptive_max_pool2d", - # "adaptive_max_pool3d", - # "affine_grid", - # "alpha_dropout", - "avg_pool1d", - "avg_pool2d", - # "avg_pool3d", - "batch_norm", - "binary_cross_entropy", - # "binary_cross_entropy_with_logits", - "celu", - # "celu_", - "conv1d", - "conv2d", - "conv3d", - "conv_transpose1d", - "conv_transpose2d", - # "conv_transpose3d", - # "cosine_embedding_loss", - # "cosine_similarity", - "cross_entropy", - "dropout", - # "dropout1d", - "dropout2d", - "dropout3d", - "elu", - # "elu_", - # "embedding", - "fold", - # "gaussian_nll_loss", - "gelu", - # "glu", - # "grid_sample", - "group_norm", - "gumbel_softmax", - "hardshrink", - "hardsigmoid", - "hardswish", - "hardtanh", - # "hardtanh_", - "hinge_embedding_loss", - "huber_loss", - "instance_norm", - "interpolate", - "kl_div", - "l1_loss", - "layer_norm", - "leaky_relu", - # "leaky_relu_", - "linear", - # "local_response_norm", - "logsigmoid", - "log_softmax", - # "lp_pool1d", - # "lp_pool2d", - # "margin_ranking_loss", - # "max_pool1d", - "max_pool2d", - # "max_pool3d", - # "mish", - "mse_loss", - # "multi_head_attention_forward", - # "multilabel_margin_loss", - # "multilabel_soft_margin_loss", - # "nll_loss", - "normalize", - "one_hot", - "pad", - # "pairwise_distance", - # "pdist", - # "pixel_shuffle", - # "pixel_unshuffle", - # "poisson_nll_loss", - # "prelu", - "relu", - "relu6", - # "relu_", - "rrelu", - # "rrelu_", - # "scaled_dot_product_attention", - "selu", - "sigmoid", - "silu", - "smooth_l1_loss", - "soft_margin_loss", - "softmax", - # "softmin", - "softplus", - "softshrink", - "softsign", - "tanh", - # "tanhshrink", - # "threshold_", - "triplet_margin_loss", - "upsample", - "upsample_bilinear", - "upsample_nearest", - ], -} - -ALL_TORCH_LAYERS = { - "torch.nn": [ - "AdaptiveAvgPool1d", - "AdaptiveAvgPool2d", - # "AdaptiveAvgPool3d", - # "AdaptiveLogSoftmaxWithLoss", - "AdaptiveMaxPool1d", - "AdaptiveMaxPool2d", - # "AdaptiveMaxPool3d", - # "AlphaDropout", - # "AvgPool1d", - "AvgPool2d", - "AvgPool3d", - # "BCELoss", - # "BCEWithLogitsLoss", - # "BatchNorm1d", - "BatchNorm2d", - "BatchNorm3d", - # "Bilinear", - # "CELU", - # "CTCLoss", - # "ChannelShuffle", - # "CircularPad1d", - # "CircularPad2d", - # "CircularPad3d", - # "ConstantPad1d", - # "ConstantPad2d", - # "ConstantPad3d", - # "Container", - # "Conv1d", - "Conv2d", - # "Conv3d", - # "ConvTranspose1d", - "ConvTranspose2d", - # "ConvTranspose3d", - # "CosineEmbeddingLoss", - # "CosineSimilarity", - # "CrossEntropyLoss", - # "CrossMapLRN2d", - # "DataParallel", - "Dropout", - # "Dropout1d", - "Dropout2d", - # "Dropout3d", - # "ELU", - # "Embedding", - # "EmbeddingBag", - "Flatten", - "Fold", - # "FractionalMaxPool2d", - # "FractionalMaxPool3d", - "GELU", - # "GLU", - "GRU", - # "GRUCell", - # "GaussianNLLLoss", - "GroupNorm", - # "Hardshrink", - "Hardsigmoid", - "Hardswish", - "Hardtanh", - # "HingeEmbeddingLoss", - # "HuberLoss", - "Identity", - # "InstanceNorm1d", - "InstanceNorm2d", - # "InstanceNorm3d", - # "KLDivLoss", - "L1Loss", - # "LPPool1d", - # "LPPool2d", - # "LPPool3d", - "LSTM", - # "LSTMCell", - "LayerNorm", - # "LazyBatchNorm1d", - # "LazyBatchNorm2d", - # "LazyBatchNorm3d", - # "LazyConv1d", - # "LazyConv2d", - # "LazyConv3d", - # "LazyConvTranspose1d", - # "LazyConvTranspose2d", - # "LazyConvTranspose3d", - # "LazyInstanceNorm1d", - # "LazyInstanceNorm2d", - # "LazyInstanceNorm3d", - # "LazyLinear", - "LeakyReLU", - "Linear", - # "LocalResponseNorm", - # "LogSigmoid", - # "LogSoftmax", - # "MSELoss", - # "MarginRankingLoss", - # "MaxPool1d", - "MaxPool2d", - # "MaxPool3d", - # "MaxUnpool1d", - # "MaxUnpool2d", - # "MaxUnpool3d", - # "Mish", - "ModuleDict", - "ModuleList", - # "MultiLabelMarginLoss", - # "MultiLabelSoftMarginLoss", - # "MultiMarginLoss", - # "MultiheadAttention", - # "NLLLoss", - # "NLLLoss2d", - "PReLU", - # "PairwiseDistance", - # "ParameterDict", - # "ParameterList", - # "PixelShuffle", - # "PixelUnshuffle", - # "PoissonNLLLoss", - # "RMSNorm", - "RNN", - # "RNNBase", - # "RNNCell", - # "RNNCellBase", - "RReLU", - "ReLU", - # "ReLU6", - # "ReflectionPad1d", - # "ReflectionPad2d", - # "ReflectionPad3d", - # "ReplicationPad1d", - # "ReplicationPad2d", - # "ReplicationPad3d", - "SELU", - "Sequential", - "SiLU", - "Sigmoid", - # "SmoothL1Loss", - # "SoftMarginLoss", - "Softmax", - # "Softmax2d", - # "Softmin", - # "Softplus", - # "Softshrink", - # "Softsign", - # "Tanh", - # "Tanhshrink", - # "Threshold", - # "Transformer", - # "TransformerDecoder", - # "TransformerDecoderLayer", - # "TransformerEncoder", - # "TransformerEncoderLayer", - # "TripletMarginLoss", - # "TripletMarginWithDistanceLoss", - # "Unflatten", - "Upsample", - "UpsamplingBilinear2d", - # "UpsamplingNearest2d", - # "ZeroPad1d", - "ZeroPad2d", - # "ZeroPad3d", - ] -} diff --git a/ivy/transpiler/caching/calculate_batch_counts.py b/ivy/transpiler/caching/calculate_batch_counts.py deleted file mode 100644 index d38357d0a260c..0000000000000 --- a/ivy/transpiler/caching/calculate_batch_counts.py +++ /dev/null @@ -1,18 +0,0 @@ -import json -import sys - - -def calculate_batch_count(matrix, batch_size): - total_items = len(matrix) - batch_count = (total_items + batch_size - 1) // batch_size - return batch_count - - -def main(matrix_json, batch_size): - matrix = json.loads(matrix_json) - batch_count = calculate_batch_count(matrix, batch_size) - print(batch_count) - - -if __name__ == "__main__": - main(sys.argv[1], int(sys.argv[2])) diff --git a/ivy/transpiler/caching/generate_matrices.py b/ivy/transpiler/caching/generate_matrices.py deleted file mode 100644 index da374256f879a..0000000000000 --- a/ivy/transpiler/caching/generate_matrices.py +++ /dev/null @@ -1,45 +0,0 @@ -import json -from typing import Dict -from ivy.transpiler.caching.assets.torch_cache_assets import ( - ALL_TORCH_FRONTEND_FUNCTIONS, - ALL_TORCH_LAYERS, -) -from ivy.transpiler.caching.assets.ivy_cache_assets import ( - ALL_IVY_FUNCTIONS, -) - - -def generate_matrix(data: Dict, source: str, target: str): - return [ - {"func_name": func_name, "module": module, "source": source, "target": target} - for module, funcs in data.items() - for func_name in funcs - ] - - -def main(matrix_type): - if matrix_type == "torch_frontend_functions_matrix": - matrix = generate_matrix( - data=ALL_TORCH_FRONTEND_FUNCTIONS, - source="torch_frontend", - target="tensorflow", - ) - elif matrix_type == "torch_layers_matrix": - matrix = generate_matrix( - data=ALL_TORCH_LAYERS, source="torch", target="tensorflow" - ) - elif matrix_type == "ivy_functions_matrix": - matrix = generate_matrix( - data=ALL_IVY_FUNCTIONS, source="ivy", target="tensorflow" - ) - else: - raise ValueError("Invalid matrix type") - - # Convert to JSON string and print it - print(json.dumps(matrix)) - - -if __name__ == "__main__": - import sys - - main(sys.argv[1]) diff --git a/ivy/transpiler/caching/merge_cache.py b/ivy/transpiler/caching/merge_cache.py deleted file mode 100644 index ff9a9a74c0cd9..0000000000000 --- a/ivy/transpiler/caching/merge_cache.py +++ /dev/null @@ -1,50 +0,0 @@ -import os -import dill -import logging - - -def merge_cache_files(): - base_dir = "/ivy" - all_caches_dir = os.path.join(base_dir, "tracer-transpiler", "all_cache_artifacts") - cache_dir = os.path.join( - base_dir, "tracer-transpiler", "ivy_repo", "ivy", "compiler", "_cache" - ) - - os.makedirs(cache_dir, exist_ok=True) - - logging.debug(f"Base directory: {base_dir}") - logging.debug(f"All caches directory: {all_caches_dir}") - logging.debug(f"Cache directory: {cache_dir}") - - pkl_files = [ - "torch_to_torch_frontend_translation_cache.pkl", - "torch_frontend_to_ivy_translation_cache.pkl", - "ivy_to_tensorflow_translation_cache.pkl", - ] - - logging.debug(f"Pickle files to process: {pkl_files}") - - for pkl_file in pkl_files: - merged_dict = {} - logging.debug(f"Processing {pkl_file}") - for root, _, files in os.walk(all_caches_dir): - if pkl_file in files: - file_path = os.path.join(root, pkl_file) - logging.debug(f"Found {pkl_file} in {file_path}") - with open(file_path, "rb") as f: - try: - data = dill.load(f) - except Exception as e: - logging.debug(e) - logging.debug(file_path) - continue - merged_dict.update(data) - - output_path = os.path.join(cache_dir, pkl_file) - with open(output_path, "wb") as f: - dill.dump(merged_dict, f) - logging.debug(f"Merged cache saved to {output_path}") - - -if __name__ == "__main__": - merge_cache_files() diff --git a/ivy/transpiler/caching/populate_cache.py b/ivy/transpiler/caching/populate_cache.py deleted file mode 100644 index 760489119629e..0000000000000 --- a/ivy/transpiler/caching/populate_cache.py +++ /dev/null @@ -1,48 +0,0 @@ -import json -import os -import importlib -import sys -import pytest -import logging - -from ivy.transpiler import transpile -from ivy.transpiler.utils.api_utils import get_function_from_modules - - -def populate_cache(func_name: str, module: str, source: str, target: str): - logging.debug(f"Populating cache for {func_name} in {source}") - os.environ["UPDATE_S2S_CACHE"] = "true" - os.environ["APPLY_TRANSPOSE_OPTIMIZATION"] = "true" - - import ivy - - ivy.set_backend(target) - - if "frontend" in source: - module = importlib.import_module(f"ivy.functional.frontends.{module}") - else: - module = importlib.import_module(module) - - # Get function dynamically from the appropriate module - func = get_function_from_modules(func_name, [module]) - transpile(func, source=source, target=target) - - -# Define a pytest function to handle each item in the batch -@pytest.mark.parametrize("item", json.loads(sys.argv[1])) -def test_populate_cache(item): - func_name = item["func_name"] - module = item["module"] - source = item["source"] - target = item["target"] - - try: - # Call the populate_cache function for each function/layer in the batch - populate_cache(func_name, module, source, target) - except Exception as e: - pytest.fail(f"Failed to populate cache for {func_name} in {module}: {e}") - - -if __name__ == "__main__": - # Run pytest programmatically - pytest.main([__file__]) diff --git a/ivy/transpiler/scripts/determine_cache_assets.sh b/ivy/transpiler/scripts/determine_cache_assets.sh deleted file mode 100644 index 4fef9e6f2b311..0000000000000 --- a/ivy/transpiler/scripts/determine_cache_assets.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -# Install dependencies -python3 -m pip install cryptography transformers black - -# Install the local ivy_repo -cd /ivy/tracer-transpiler/ivy_repo -python3 -m pip install --user -e . - -cd .. - -# Generate matrices using Python script -cp source_to_source_translator/caching/generate_matrices.py . -torch_frontend_functions_matrix=$(python3 generate_matrices.py torch_frontend_functions_matrix | tr -d '\n') -torch_layers_matrix=$(python3 generate_matrices.py torch_layers_matrix | tr -d '\n') -ivy_functions_matrix=$(python3 generate_matrices.py ivy_functions_matrix | tr -d '\n') - -# Calculate batch counts using Python script -cp source_to_source_translator/caching/calculate_batch_counts.py . -torch_frontend_functions_batch_count=$(python3 calculate_batch_counts.py "$torch_frontend_functions_matrix" "$TORCH_FRONTEND_FUNCS_BATCH_SIZE") -torch_layers_batch_count=$(python3 calculate_batch_counts.py "$torch_layers_matrix" "$TORCH_LAYERS_BATCH_SIZE") -ivy_functions_batch_count=$(python3 calculate_batch_counts.py "$ivy_functions_matrix" "$IVY_FUNCS_BATCH_SIZE") - -# Write outputs to GITHUB_OUTPUT -if [ -n "$GITHUB_OUTPUT" ]; then - echo -n "torch_frontend_functions_matrix=" >> "$GITHUB_OUTPUT" - echo "$torch_frontend_functions_matrix" >> "$GITHUB_OUTPUT" - echo -n "torch_layers_matrix=" >> "$GITHUB_OUTPUT" - echo "$torch_layers_matrix" >> "$GITHUB_OUTPUT" - echo -n "ivy_functions_matrix=" >> "$GITHUB_OUTPUT" - echo "$ivy_functions_matrix" >> "$GITHUB_OUTPUT" - echo "torch_frontend_functions_batch_count=$torch_frontend_functions_batch_count" >> "$GITHUB_OUTPUT" - echo "torch_layers_batch_count=$torch_layers_batch_count" >> "$GITHUB_OUTPUT" - echo "ivy_functions_batch_count=$ivy_functions_batch_count" >> "$GITHUB_OUTPUT" -fi diff --git a/ivy/transpiler/scripts/determine_main_tests.sh b/ivy/transpiler/scripts/determine_main_tests.sh deleted file mode 100644 index 5993160f4829a..0000000000000 --- a/ivy/transpiler/scripts/determine_main_tests.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -# install dependencies -python3 -m pip install cryptography -python3 -m pip install dill -python3 -m pip install black -python3 -m pip install pytest - -# TODO: update repo path -# install the local ivy_repo -cd /ivy/tracer-transpiler/ivy_repo -python3 -m pip install --user -e . -python3 -m pip install -r tests/source2source/requirements.txt - -# Python script to generate the matrix -python3 <" - exit 1 -fi - -# Parse the batch JSON -batch=$1 - -# Pass the entire batch to the populate_cache.py script -python3 populate_cache.py "$batch" diff --git a/ivy/transpiler/scripts/run_dummy_determine_cache_assets.sh b/ivy/transpiler/scripts/run_dummy_determine_cache_assets.sh deleted file mode 100644 index 80f3adf6acb4a..0000000000000 --- a/ivy/transpiler/scripts/run_dummy_determine_cache_assets.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -# Emulate GitHub Actions environment variables -export GITHUB_OUTPUT="/tmp/github_output.txt" # Temp file to simulate GitHub Actions output file -export TORCH_FRONTEND_FUNCS_BATCH_SIZE=10 -export TORCH_LAYERS_BATCH_SIZE=5 -export IVY_FUNCS_BATCH_SIZE=15 - -# Ensure the GITHUB_OUTPUT file is empty at the start -echo -n "" > $GITHUB_OUTPUT - -# Run the determine_cache_assets.sh script -./source_to_source_translator/scripts/determine_cache_assets.sh - -# Print the contents of the GITHUB_OUTPUT file to the console -echo "GITHUB_OUTPUT content:" -cat $GITHUB_OUTPUT