diff --git a/publish/electron-builder.json b/publish/electron-builder.json index 103ad1f003..129b42bfa1 100644 --- a/publish/electron-builder.json +++ b/publish/electron-builder.json @@ -11,6 +11,7 @@ { "ext": "cmf", "name": "CNTK Model" }, { "ext": "dlc", "name": "DLC Model" }, { "ext": "dnn", "name": "CNTK Model" }, + { "ext": "espdl", "name": "ESPDL Model" }, { "ext": "gguf", "name": "GGUF Model" }, { "ext": "h5", "name": "Keras Model" }, { "ext": "hd5", "name": "Keras Model" }, diff --git a/source/base.js b/source/base.js index 083319a121..a13491b554 100644 --- a/source/base.js +++ b/source/base.js @@ -1273,7 +1273,7 @@ base.Metadata = class { 'pkl', 'pickle', 'joblib', 'safetensors', 'ptl', 't7', 'dlc', 'uff', 'armnn', 'kann', 'kgraph', - 'mnn', 'ms', 'ncnn', 'om', 'tm', 'mge', 'tmfile', 'tnnproto', 'xmodel', 'kmodel', 'rknn', + 'mnn', 'ms', 'ncnn', 'om', 'tm', 'mge', 'tmfile', 'tnnproto', 'xmodel', 'kmodel', 'rknn', 'espdl', 'tar', 'zip' ]; } diff --git a/source/espdl-metadata.json b/source/espdl-metadata.json new file mode 100644 index 0000000000..f03184bc59 --- /dev/null +++ b/source/espdl-metadata.json @@ -0,0 +1,282 @@ +[ + { + "name": "Add", + "module": "espdl", + "version": 1, + "description": "Element-wise addition of two tensors.", + "inputs": [ + { "name": "A", "description": "First input tensor" }, + { "name": "B", "description": "Second input tensor" } + ], + "outputs": [ + { "name": "C", "description": "Output tensor" } + ], + "category": "Tensor" + }, + { + "name": "AveragePool", + "module": "espdl", + "version": 1, + "description": "Average pooling operation.", + "inputs": [ + { "name": "input", "description": "Input tensor" } + ], + "outputs": [ + { "name": "output", "description": "Output tensor" } + ], + "category": "Pool" + }, + { + "name": "BatchNormalization", + "module": "espdl", + "version": 1, + "description": "Batch normalization.", + "inputs": [ + { "name": "X", "description": "Input data tensor" }, + { "name": "scale", "description": "Scale tensor" }, + { "name": "B", "description": "Bias tensor" }, + { "name": "mean", "description": "Mean tensor" }, + { "name": "var", "description": "Variance tensor" } + ], + "outputs": [ + { "name": "Y", "description": "Output data tensor" }, + { "name": "mean", "description": "Updated mean tensor (optional)" }, + { "name": "var", "description": "Updated variance tensor (optional)" }, + { "name": "saved_mean", "description": "Saved mean tensor (optional)" }, + { "name": "saved_var", "description": "Saved variance tensor (optional)" } + ], + "category": "Normalization" + }, + { + "name": "Clip", + "module": "espdl", + "version": 1, + "category": "Activation", + "description": "Clip operator limits values to a specified range.", + "inputs": [ + { "name": "input", "description": "Input tensor" } + ], + "outputs": [ + { "name": "output", "description": "Clipped output tensor" } + ] + }, + { + "name": "Concat", + "module": "espdl", + "version": 1, + "description": "Concatenates tensors along a given axis.", + "inputs": [ + { "name": "inputs", "list": true, "description": "Input tensors to concatenate" } + ], + "outputs": [ + { "name": "output", "description": "Concatenated output tensor" } + ], + "category": "Tensor" + }, + { + "name": "Conv", + "module": "espdl", + "version": 1, + "description": "Convolution operator. Applies a convolution filter to the input.", + "inputs": [ + { "name": "input", "description": "Input feature map" }, + { "name": "weight", "description": "Convolution kernel weights" }, + { "name": "bias", "option": "optional", "description": "Bias values (optional)" } + ], + "outputs": [ + { "name": "output", "description": "Output feature map" } + ], + "category": "Layer" + }, + { + "name": "Gemm", + "module": "espdl", + "version": 1, + "description": "General matrix multiplication: alpha * A * B + beta * C", + "inputs": [ + { "name": "A", "description": "Input tensor A" }, + { "name": "B", "description": "Input tensor B" }, + { "name": "C", "option": "optional", "description": "Input tensor C (optional)" } + ], + "outputs": [ + { "name": "Y", "description": "Output tensor" } + ], + "category": "Layer" + }, + { + "name": "LeakyRelu", + "module": "espdl", + "version": 1, + "category": "Activation", + "description": "Leaky Rectified Linear Unit activation function.", + "inputs": [ + { "name": "input", "description": "Input tensor" } + ], + "outputs": [ + { "name": "output", "description": "Output tensor" } + ] + }, + { + "name": "MaxPool", + "module": "espdl", + "version": 1, + "description": "Max pooling operation.", + "inputs": [ + { "name": "input", "description": "Input tensor" } + ], + "outputs": [ + { "name": "output", "description": "Output tensor" } + ], + "category": "Pool" + }, + { + "name": "Mul", + "module": "espdl", + "version": 1, + "description": "Element-wise multiplication of two tensors.", + "inputs": [ + { "name": "A", "description": "First input tensor" }, + { "name": "B", "description": "Second input tensor" } + ], + "outputs": [ + { "name": "C", "description": "Output tensor" } + ], + "category": "Tensor" + }, + { + "name": "Pad", + "module": "espdl", + "version": 1, + "category": "Tensor", + "description": "Pad operator adds padding to tensor dimensions.", + "inputs": [ + { "name": "input", "description": "Input tensor" }, + { "name": "pads", "description": "Padding values" } + ], + "outputs": [ + { "name": "output", "description": "Padded output tensor" } + ] + }, + { + "name": "Relu", + "module": "espdl", + "version": 1, + "description": "Rectified Linear Unit activation function.", + "inputs": [ + { "name": "input", "description": "Input tensor" } + ], + "outputs": [ + { "name": "output", "description": "Output tensor" } + ], + "category": "Activation" + }, + { + "name": "RequantizeLinear", + "module": "espdl", + "version": 1, + "category": "Quantization", + "description": "Requantize linear quantization operator.", + "inputs": [ + { "name": "input", "description": "Input tensor to requantize" }, + { "name": "scale", "description": "Scale for requantization" }, + { "name": "zero_point", "description": "Zero point for requantization" } + ], + "outputs": [ + { "name": "output", "description": "Requantized output tensor" } + ] + }, + { + "name": "Reshape", + "module": "espdl", + "version": 1, + "description": "Reshapes a tensor to a new shape.", + "inputs": [ + { "name": "data", "description": "Input tensor" }, + { "name": "shape", "description": "New shape" } + ], + "outputs": [ + { "name": "reshaped", "description": "Reshaped output tensor" } + ], + "category": "Shape" + }, + { + "name": "Resize", + "module": "espdl", + "version": 1, + "category": "Data", + "description": "Resize operator for spatial dimensions.", + "inputs": [ + { "name": "input", "description": "Input tensor" }, + { "name": "scales", "description": "Scale factors for each dimension" } + ], + "outputs": [ + { "name": "output", "description": "Resized output tensor" } + ] + }, + { + "name": "Sigmoid", + "module": "espdl", + "version": 1, + "description": "Sigmoid activation function.", + "inputs": [ + { "name": "input", "description": "Input tensor" } + ], + "outputs": [ + { "name": "output", "description": "Output tensor" } + ], + "category": "Activation" + }, + { + "name": "Softmax", + "module": "espdl", + "version": 1, + "description": "Softmax activation function.", + "inputs": [ + { "name": "input", "description": "Input tensor" } + ], + "outputs": [ + { "name": "output", "description": "Output tensor" } + ], + "category": "Activation" + }, + { + "name": "Split", + "module": "espdl", + "version": 1, + "description": "Splits a tensor into multiple tensors along a given axis.", + "inputs": [ + { "name": "input", "description": "Input tensor to split" }, + { "name": "split", "description": "Optional list of split sizes or number of splits" } + ], + "outputs": [ + { "name": "outputs", "list": true, "description": "Output tensors" } + ], + "category": "Tensor" + }, + { + "name": "Tanh", + "module": "espdl", + "version": 1, + "description": "Hyperbolic tangent activation function.", + "inputs": [ + { "name": "input", "description": "Input tensor" } + ], + "outputs": [ + { "name": "output", "description": "Output tensor" } + ], + "category": "Activation" + }, + { + "name": "Transpose", + "module": "espdl", + "version": 1, + "description": "Transposes the dimensions of a tensor.", + "inputs": [ + { "name": "data", "description": "Input tensor" } + ], + "outputs": [ + { "name": "transposed", "description": "Transposed output tensor" } + ], + "category": "Tensor" + } +] \ No newline at end of file diff --git a/source/espdl-schema.js b/source/espdl-schema.js new file mode 100644 index 0000000000..4a398ec9f2 --- /dev/null +++ b/source/espdl-schema.js @@ -0,0 +1,329 @@ + +export const espdl = {}; + +espdl.Version = { + _START_VERSION: 0, + IR_VERSION_2023_12_22: 1 +}; + +espdl.AttributeType = { + UNDEFINED: 0, + FLOAT: 1, + INT: 2, + STRING: 3, + TENSOR: 4, + GRAPH: 5, + FLOATS: 6, + INTS: 7, + STRINGS: 8, + TENSORS: 9, + GRAPHS: 10, + TYPE_FBS: 11, + TYPE_FBSS: 12 +}; + +espdl.TensorDataType = { + UNDEFINED: 0, + FLOAT: 1, + UINT8: 2, + INT8: 3, + UINT16: 4, + INT16: 5, + INT32: 6, + INT64: 7, + STRING: 8, + BOOL: 9, + FLOAT16: 10, + DOUBLE: 11, + UINT32: 12, + UINT64: 13 +}; + +espdl.DataLocation = { + DEFAULT: 0, + EXTERNAL: 1 +}; + +espdl.AttributeF = class AttributeF { + + static decode(reader, position) { + const $ = new espdl.AttributeF(); + $.f = reader.float32(position + 0); + return $; + } +}; + +espdl.AttributeI = class AttributeI { + + static decode(reader, position) { + const $ = new espdl.AttributeI(); + $.i = reader.int64(position + 0); + return $; + } +}; + +espdl.Attribute = class Attribute { + + static decode(reader, position) { + const $ = new espdl.Attribute(); + $.name = reader.string_(position, 4, null); + $.ref_attr_name = reader.string_(position, 6, null); + $.doc_string = reader.string_(position, 8, null); + $.attr_type = reader.int32_(position, 10, 0); + $.f = reader.struct(position, 12, espdl.AttributeF); + $.i = reader.struct(position, 14, espdl.AttributeI); + $.s = reader.array(position, 16, Uint8Array); + $.t = reader.table(position, 18, espdl.Tensor); + $.g = reader.table(position, 20, espdl.Graph); + $.tp = reader.table(position, 22, espdl.TypeInfo); + $.floats = reader.array(position, 24, Float32Array); + $.ints = reader.int64s_(position, 26); + $.strings = reader.strings_(position, 28); + $.tensors = reader.tables(position, 30, espdl.Tensor); + $.graphs = reader.tables(position, 32, espdl.Graph); + $.type_protos = reader.tables(position, 34, espdl.TypeInfo); + return $; + } +}; + +espdl.ValueInfo = class ValueInfo { + + static decode(reader, position) { + const $ = new espdl.ValueInfo(); + $.name = reader.string_(position, 4, null); + $.value_info_type = reader.table(position, 6, espdl.TypeInfo); + $.doc_string = reader.string_(position, 8, null); + $.exponents = reader.int64s_(position, 10); + return $; + } +}; + +espdl.Node = class Node { + + static decode(reader, position) { + const $ = new espdl.Node(); + $.input = reader.strings_(position, 4); + $.output = reader.strings_(position, 6); + $.name = reader.string_(position, 8, null); + $.op_type = reader.string_(position, 10, null); + $.domain = reader.string_(position, 12, null); + $.attribute = reader.tables(position, 14, espdl.Attribute); + $.doc_string = reader.string_(position, 16, null); + return $; + } +}; + +espdl.Model = class Model { + + static create(reader) { + return espdl.Model.decode(reader, reader.root); + } + + static decode(reader, position) { + const $ = new espdl.Model(); + $.ir_version = reader.int32_(position, 4, 0); + $.opset_import = reader.tables(position, 6, espdl.OperatorSetId); + $.producer_name = reader.string_(position, 8, null); + $.producer_version = reader.string_(position, 10, null); + $.domain = reader.string_(position, 12, null); + $.model_version = reader.int64_(position, 14, 0n); + $.doc_string = reader.string_(position, 16, null); + $.graph = reader.table(position, 18, espdl.Graph); + $.metadata_props = reader.tables(position, 20, espdl.StringStringEntry); + $.functions = reader.tables(position, 22, espdl.Function); + return $; + } +}; + +espdl.StringStringEntry = class StringStringEntry { + + static decode(reader, position) { + const $ = new espdl.StringStringEntry(); + $.key = reader.string_(position, 4, null); + $.value = reader.string_(position, 6, null); + return $; + } +}; + +espdl.TensorAnnotation = class TensorAnnotation { + + static decode(reader, position) { + const $ = new espdl.TensorAnnotation(); + $.tensor_name = reader.string_(position, 4, null); + $.quant_parameter_tensor_names = reader.tables(position, 6, espdl.StringStringEntry); + return $; + } +}; + +espdl.Graph = class Graph { + + static decode(reader, position) { + const $ = new espdl.Graph(); + $.node = reader.tables(position, 4, espdl.Node); + $.name = reader.string_(position, 6, null); + $.initializer = reader.tables(position, 8, espdl.Tensor); + $.doc_string = reader.string_(position, 10, null); + $.input = reader.tables(position, 12, espdl.ValueInfo); + $.output = reader.tables(position, 14, espdl.ValueInfo); + $.value_info = reader.tables(position, 16, espdl.ValueInfo); + $.quantization_annotation = reader.tables(position, 18, espdl.TensorAnnotation); + $.test_inputs_value = reader.tables(position, 20, espdl.Tensor); + $.test_outputs_value = reader.tables(position, 22, espdl.Tensor); + return $; + } +}; + +espdl.AlignedBytes = class AlignedBytes { + + static decode() { + const $ = new espdl.AlignedBytes(); + $.bytes = undefined; // not implemented + return $; + } +}; + +espdl.Tensor = class Tensor { + + static decode(reader, position) { + const $ = new espdl.Tensor(); + $.dims = reader.int64s_(position, 4); + $.data_type = reader.int32_(position, 6, 0); + $.float_data = reader.array(position, 8, Float32Array); + $.int32_data = reader.array(position, 10, Int32Array); + $.string_data = reader.strings_(position, 12); + $.int64_data = reader.int64s_(position, 14); + $.name = reader.string_(position, 16, null); + $.doc_string = reader.string_(position, 18, null); + $.raw_data = reader.structs(position, 20, espdl.AlignedBytes); + $.external_data = reader.tables(position, 22, espdl.StringStringEntry); + $.data_location = reader.int32_(position, 24, 0); + $.double_data = reader.array(position, 26, Float64Array); + $.uint64_data = reader.uint64s_(position, 28); + $.exponents = reader.int64s_(position, 30); + return $; + } +}; + +espdl.TensorShape = class TensorShape { + + static decode(reader, position) { + const $ = new espdl.TensorShape(); + $.dim = reader.tables(position, 4, espdl.Dimension); + return $; + } +}; + +espdl.Dimension = class Dimension { + + static decode(reader, position) { + const $ = new espdl.Dimension(); + $.value = reader.table(position, 4, espdl.DimensionValue); + $.denotation = reader.string_(position, 6, null); + return $; + } +}; + +espdl.DimensionValueType = { + UNKNOWN: 0, + VALUE: 1, + PARAM: 2 +}; + +espdl.DimensionValue = class DimensionValue { + + static decode(reader, position) { + const $ = new espdl.DimensionValue(); + $.dim_type = reader.int8_(position, 4, 0); + $.dim_value = reader.int64_(position, 6, 0n); + $.dim_param = reader.string_(position, 8, null); + return $; + } +}; + +espdl.TensorTypeAndShape = class TensorTypeAndShape { + + static decode(reader, position) { + const $ = new espdl.TensorTypeAndShape(); + $.elem_type = reader.int32_(position, 4, 0); + $.shape = reader.table(position, 6, espdl.TensorShape); + return $; + } +}; + +espdl.SequenceType = class SequenceType { + + static decode(reader, position) { + const $ = new espdl.SequenceType(); + $.elem_type = reader.table(position, 4, espdl.TypeInfo); + return $; + } +}; + +espdl.MapType = class MapType { + + static decode(reader, position) { + const $ = new espdl.MapType(); + $.key_type = reader.int32_(position, 4, 0); + $.value_type = reader.table(position, 6, espdl.TypeInfo); + return $; + } +}; + +espdl.OptionalType = class OptionalType { + + static decode(reader, position) { + const $ = new espdl.OptionalType(); + $.elem_type = reader.table(position, 4, espdl.TypeInfo); + return $; + } +}; + +espdl.TypeInfoValue = class { + + static decode(reader, position, type) { + switch (type) { + case 1: return espdl.TensorTypeAndShape.decode(reader, position); + case 2: return espdl.SequenceType.decode(reader, position); + case 3: return espdl.MapType.decode(reader, position); + case 4: return espdl.OptionalType.decode(reader, position); + default: return undefined; + } + } +}; + +espdl.TypeInfo = class TypeInfo { + + static decode(reader, position) { + const $ = new espdl.TypeInfo(); + $.value = reader.union(position, 4, espdl.TypeInfoValue); + $.denotation = reader.string_(position, 8, null); + return $; + } +}; + +espdl.OperatorSetId = class OperatorSetId { + + static decode(reader, position) { + const $ = new espdl.OperatorSetId(); + $.domain = reader.string_(position, 4, null); + $.version = reader.int64_(position, 6, 0n); + return $; + } +}; + +espdl.Function = class Function { + + static decode(reader, position) { + const $ = new espdl.Function(); + $.name = reader.string_(position, 4, null); + $.input = reader.strings_(position, 6); + $.output = reader.strings_(position, 8); + $.attribute = reader.strings_(position, 10); + $.attribute_proto = reader.tables(position, 12, espdl.Attribute); + $.node = reader.tables(position, 14, espdl.Node); + $.doc_string = reader.string_(position, 16, null); + $.opset_import = reader.tables(position, 18, espdl.OperatorSetId); + $.domain = reader.string_(position, 20, null); + return $; + } +}; diff --git a/source/espdl.js b/source/espdl.js new file mode 100644 index 0000000000..b6e6eda987 --- /dev/null +++ b/source/espdl.js @@ -0,0 +1,575 @@ +import * as flatbuffers from './flatbuffers.js'; + +const espdl = {}; + +espdl.ModelFactory = class { + + async match(context) { + // Check by file extension + const identifier = context.identifier; + const extension = identifier.lastIndexOf('.') > 0 ? identifier.split('.').pop().toLowerCase() : ''; + if (extension === 'espdl') { + // Check for EDL2 header using stream directly + const stream = context.stream; + if (stream && stream.length >= 16) { + const buffer = stream.peek(16); // Peek first 16 bytes + const header = String.fromCharCode(...buffer.slice(0, 4)); + if (header === 'EDL2') { + // We'll set a custom context type, but we need to skip header later + // For now, just indicate match + return context.set('espdl.binary', null); // We'll read the buffer in open method + } + } + } + + return null; + } + + async open(context) { + try { + const schemaModule = await context.require('./espdl-schema'); + espdl.schema = schemaModule.espdl; + if (!espdl.schema || !espdl.schema.Model) { + throw new espdl.Error(`Failed to load ESPDL schema: Model is ${espdl.schema?.Model ? 'defined' : 'undefined'}`); + } + } catch (error) { + throw new espdl.Error(`Failed to load ESPDL schema: ${error.message}`); + } + let model = null; + if (context.type === 'espdl.binary') { + // Read from stream directly + const stream = context.stream; + if (!stream) { + throw new espdl.Error('No stream available for ESP-DL binary file.'); + } + + // Read 16-byte header + const headerBuffer = stream.read(16); + if (headerBuffer.length < 16) { + throw new espdl.Error('Invalid ESP-DL file: header too short.'); + } + + // Verify magic + const magic = String.fromCharCode(...headerBuffer.slice(0, 4)); + if (magic !== 'EDL2') { + throw new espdl.Error(`Invalid ESP-DL magic: ${magic}, expected EDL2`); + } + + // Read the rest of the file as FlatBuffers data + const data = stream.read(stream.length - stream.position); + + // Create flatbuffers reader + const reader = flatbuffers.BinaryReader.open(data); + if (!reader) { + throw new espdl.Error('Invalid FlatBuffers data after header.'); + } + try { + model = espdl.schema.Model.create(reader); + } catch (error) { + const message = error && error.message ? error.message : error.toString(); + throw new espdl.Error(`File format is not espdl.Model (${message.replace(/\.$/, '')}).`); + } + } else { + throw new espdl.Error(`Unsupported ESP-DL format '${context.type}'.`); + } + const stream = context.stream; + const metadata = await espdl.Metadata.open(context); + return new espdl.Model(metadata, model, stream); + } +}; + +espdl.Model = class { + + constructor(metadata, model, stream) { + this.format = 'espdl'; + this.description = model.doc_string || ''; + this.modules = []; + this.metadata = []; + // Parse model metadata props + if (model.metadata_props) { + for (const prop of model.metadata_props) { + this.metadata.push(new espdl.Argument(prop.key, prop.value)); + } + } + // Process graph + const graph = model.graph; + if (graph) { + const graphObj = new espdl.Graph(metadata, graph, model, stream); + this.modules.push(graphObj); + } + } +}; + +espdl.Graph = class { + + constructor(metadata, graph) { + this.name = graph.name || ''; + this.inputs = []; + this.outputs = []; + this.nodes = []; + this.signatures = []; + + // Create context for tensor management + const context = new espdl.Context(graph); + + // Process nodes with context and metadata + if (graph.node) { + for (let i = 0; i < graph.node.length; i++) { + const node = graph.node[i]; + const nodeObj = new espdl.Node(metadata, context, node, i.toString()); + this.nodes.push(nodeObj); + } + } + + // Process inputs (only non-initializer inputs) + if (graph.input) { + for (let i = 0; i < graph.input.length; i++) { + const valueInfo = graph.input[i]; + const tensor = context.initializer(valueInfo.name); + if (!tensor) { + const value = context.value(valueInfo.name); + const values = value ? [value] : []; + const argument = new espdl.Argument(valueInfo.name, values); + this.inputs.push(argument); + } + } + } + + // Process outputs + if (graph.output) { + for (let i = 0; i < graph.output.length; i++) { + const valueInfo = graph.output[i]; + const value = context.value(valueInfo.name); + const values = value ? [value] : []; + const argument = new espdl.Argument(valueInfo.name, values); + this.outputs.push(argument); + } + } + } +}; + +espdl.Node = class { + + constructor(metadata, context, node, identifier) { + this.name = node.name || ''; + this.identifier = identifier; + + // Get operator type from metadata + const opType = node.op_type; + this.type = metadata ? metadata.type('espdl', opType) : null; + if (!this.type) { + this.type = { name: opType }; + } + + this.inputs = []; + this.outputs = []; + this.attributes = []; + + // Map inputs using context and metadata + if (node.input) { + for (let i = 0; i < node.input.length;) { + const inputMeta = this.type && Array.isArray(this.type.inputs) && i < this.type.inputs.length ? this.type.inputs[i] : { name: i.toString() }; + const count = inputMeta.list ? node.input.length - i : 1; + const list = node.input.slice(i, i + count); + const values = list.map((inputName) => { + if (!inputName) { + return null; // Skip empty names + } + return context.value(inputName); + }).filter((v) => v); + const argument = new espdl.Argument(inputMeta.name, values); + this.inputs.push(argument); + i += count; + } + } + + // Map outputs using context and metadata + if (node.output) { + for (let i = 0; i < node.output.length;) { + const outputMeta = this.type && Array.isArray(this.type.outputs) && i < this.type.outputs.length ? this.type.outputs[i] : { name: i.toString() }; + const count = outputMeta.list ? node.output.length - i : 1; + const list = node.output.slice(i, i + count); + const values = list.map((outputName) => { + // Get or create tensor for output + if (!outputName) { + return null; // Skip empty names + } + let tensor = context.value(outputName); + if (!tensor) { + // Check if we have value_info for this tensor + // If not, create a basic tensor with just the name + const tensorObj = { name: outputName }; + tensor = new espdl.Tensor(context._values.size, tensorObj, null); + context._values.set(outputName, tensor); + } + return tensor; + }).filter((v) => v); + const argument = new espdl.Argument(outputMeta.name, values); + this.outputs.push(argument); + i += count; + } + } + + // Process attributes + if (node.attribute) { + for (const attr of node.attribute) { + const attrObj = new espdl.Attribute(attr); + this.attributes.push(attrObj); + } + } + } +}; + +espdl.Tensor = class { + + constructor(index, tensor) { + this.identifier = index.toString(); + this.name = tensor.name || ''; + if (this.name === undefined) { + this.name = ''; + } + this.type = tensor.data_type === undefined ? null : new espdl.TensorType(tensor); + this.category = ''; + this.encoding = '<'; // little-endian assumption + // load tensor data from stream if external + this._data = null; + // Reference to initializer tensor (for weight display) + this.initializer = null; // Will be set to self for initializers + + // For initializers or tensors with data + if (tensor.raw_data && tensor.raw_data.length > 0) { + // raw_data is array of AlignedBytes + // Note: AlignedBytes.bytes is undefined in current implementation + // So we cannot extract data from raw_data + this._data = null; + } else if (tensor.float_data && tensor.float_data.length > 0) { + this._data = new Float32Array(tensor.float_data); + } else if (tensor.int32_data && tensor.int32_data.length > 0) { + this._data = new Int32Array(tensor.int32_data); + } else if (tensor.int64_data && tensor.int64_data.length > 0) { + this._data = new BigInt64Array(tensor.int64_data); + } else if (tensor.string_data && tensor.string_data.length > 0) { + this._data = tensor.string_data; + } + + // For intermediate tensors (outputs), we might not have type info + if (!this.type) { + let shapeDims = []; + // Check if we have dims directly on tensor + if (tensor.dims) { + shapeDims = Array.from(tensor.dims).map((d) => Number(d)); + } else if (tensor.value_info_type !== undefined) { + // Check if this is a ValueInfo object + const dims = espdl.Utility.getShapeFromValueInfo(tensor); + if (dims) { + shapeDims = dims; + } + } + + if (shapeDims.length > 0) { + // Create a basic type with shape only + this.type = { + _dataType: '?', + _shape: new espdl.TensorShape(shapeDims), + get dataType() { + return this._dataType; + }, + get shape() { + return this._shape; + }, + toString() { + return this._dataType + this._shape.toString(); + } + }; + } + } + } + + get values() { + return this._data; + } +}; + +espdl.TensorType = class { + + constructor(tensor) { + // Check if this is a ValueInfo object (has value_info_type) + if (tensor.value_info_type === undefined) { + // Regular tensor object + this._dataType = tensor.data_type === undefined ? '?' : espdl.Utility.dataType(tensor.data_type); + this._shape = new espdl.TensorShape(tensor.dims ? Array.from(tensor.dims).map((d) => Number(d)) : []); + } else { + // Extract data type from ValueInfo + const dataType = espdl.Utility.getDataTypeFromValueInfo(tensor); + this._dataType = dataType === undefined ? '?' : espdl.Utility.dataType(dataType); + + // Extract shape from ValueInfo + const shapeDims = espdl.Utility.getShapeFromValueInfo(tensor); + this._shape = new espdl.TensorShape(shapeDims || []); + } + } + + get dataType() { + return this._dataType; + } + + get shape() { + return this._shape; + } + + toString() { + return this._dataType + this._shape.toString(); + } +}; + +espdl.TensorShape = class { + + constructor(dimensions) { + this._dimensions = dimensions; + } + + get dimensions() { + return this._dimensions; + } + + toString() { + if (!this._dimensions || this._dimensions.length === 0) { + return ''; + } + return `[${this._dimensions.map((dimension) => dimension.toString()).join(',')}]`; + } +}; + +espdl.Attribute = class { + + constructor(attr) { + this.name = attr.name || ''; + this.value = null; + this.type = null; + this.visible = true; + // Convert attribute value based on attr_type + switch (attr.attr_type) { + case espdl.schema.AttributeType.FLOAT: + this.value = attr.f ? attr.f.f : 0; + break; + case espdl.schema.AttributeType.INT: + this.value = attr.i ? Number(attr.i.i) : 0; + break; + case espdl.schema.AttributeType.STRING: + this.value = attr.s ? new TextDecoder('utf-8').decode(attr.s) : ''; + break; + case espdl.schema.AttributeType.TENSOR: + this.value = ''; + break; + case espdl.schema.AttributeType.FLOATS: + this.value = attr.floats ? Array.from(attr.floats) : []; + break; + case espdl.schema.AttributeType.INTS: + this.value = attr.ints ? Array.from(attr.ints).map((i) => Number(i)) : []; + break; + case espdl.schema.AttributeType.STRINGS: + this.value = attr.strings ? attr.strings.map((s) => new TextDecoder('utf-8').decode(s)) : []; + break; + default: + this.value = ''; + } + } +}; + +espdl.Argument = class { + + constructor(name, value, type, visible) { + this.name = name; + this.value = value; + this.type = type || null; + this.visible = visible !== false; + } +}; + +espdl.Utility = class { + + static dataType(type) { + if (!espdl.Utility._tensorTypes) { + espdl.Utility._tensorTypes = new Map(Object.entries(espdl.schema.TensorDataType).map(([key, value]) => [value, key.toLowerCase()])); + } + return espdl.Utility._tensorTypes.has(type) ? espdl.Utility._tensorTypes.get(type) : '?'; + } + + // Extract shape from ValueInfo object + static getShapeFromValueInfo(valueInfo) { + if (!valueInfo || !valueInfo.value_info_type) { + return null; + } + const typeInfo = valueInfo.value_info_type; + if (!typeInfo.value) { + return null; + } + // Check if value is a TensorTypeAndShape object (union type 1) + // In FlatBuffers, union objects don't have a 'type' property + // They are directly the object if the union type matches + const tensorType = typeInfo.value; + if (!tensorType.shape) { + return null; + } + const shape = tensorType.shape; + if (!shape.dim || shape.dim.length === 0) { + return []; + } + // Convert Dimension objects to dimension values + const dimensions = []; + for (const dim of shape.dim) { + if (dim && dim.value) { + const dimValue = dim.value; + // DimensionValueType enum values: UNKNOWN=0, VALUE=1, PARAM=2 + if (dimValue.dim_type === 1) { // VALUE + dimensions.push(Number(dimValue.dim_value)); + } else if (dimValue.dim_type === 2) { // PARAM + dimensions.push(dimValue.dim_param || '?'); + } else { + dimensions.push('?'); + } + } else { + dimensions.push('?'); + } + } + return dimensions; + } + + // Extract data type from ValueInfo object + static getDataTypeFromValueInfo(valueInfo) { + if (!valueInfo || !valueInfo.value_info_type) { + return undefined; + } + const typeInfo = valueInfo.value_info_type; + if (!typeInfo.value) { + return undefined; + } + const tensorType = typeInfo.value; + return tensorType.elem_type === undefined ? undefined : tensorType.elem_type; + } +}; + +espdl.Metadata = class { + + static async open(context) { + if (!espdl.Metadata._metadata) { + let data = null; + try { + data = await context.request('espdl-metadata.json'); + } catch { + // continue regardless of error + } + espdl.Metadata._metadata = new espdl.Metadata(data); + } + return espdl.Metadata._metadata; + } + + constructor(data) { + this._types = new Map(); + if (data) { + const types = JSON.parse(data); + for (const type of types) { + if (!this._types.has(type.module)) { + this._types.set(type.module, new Map()); + } + const typesByModule = this._types.get(type.module); + if (!typesByModule.has(type.name)) { + typesByModule.set(type.name, []); + } + typesByModule.get(type.name).push(type); + } + } + } + + type(domain, name) { + domain = domain || 'espdl'; + if (this._types.has(domain)) { + const types = this._types.get(domain); + if (types.has(name)) { + // Return the highest version + const typeList = types.get(name); + return typeList.reduce((max, current) => + (current.version > max.version) ? current : max, typeList[0]); + } + } + return null; + } +}; + +espdl.Context = class { + + constructor(graph) { + this._initializers = new Map(); + this._tensors = new Map(); + this._values = new Map(); + + // initializers + if (graph.initializer) { + for (let i = 0; i < graph.initializer.length; i++) { + const tensor = graph.initializer[i]; + const tensorName = tensor.name || ''; + if (tensorName) { + const tensorObj = new espdl.Tensor(i, tensor, null); + tensorObj.initializer = tensorObj; // Self-reference for initializer + this._initializers.set(tensorName, tensorObj); + this._values.set(tensorName, tensorObj); + } + } + } + + // inputs + if (graph.input) { + for (const valueInfo of graph.input) { + const valueName = valueInfo.name || ''; + if (valueName && !this._values.has(valueName)) { + const tensor = new espdl.Tensor(this._values.size, valueInfo, null); + this._values.set(valueName, tensor); + } + } + } + + // outputs + if (graph.output) { + for (const valueInfo of graph.output) { + const valueName = valueInfo.name || ''; + if (valueName && !this._values.has(valueName)) { + const tensor = new espdl.Tensor(this._values.size, valueInfo, null); + this._values.set(valueName, tensor); + } + } + } + + // value_info (intermediate tensors with type information) + if (graph.value_info) { + for (const valueInfo of graph.value_info) { + const valueName = valueInfo.name || ''; + if (valueName && !this._values.has(valueName)) { + const tensor = new espdl.Tensor(this._values.size, valueInfo, null); + this._values.set(valueName, tensor); + } + } + } + } + + value(name) { + return this._values.get(name) || null; + } + + tensor(name) { + return this._values.get(name) || null; + } + + initializer(name) { + return this._initializers.get(name) || null; + } +}; + +espdl.Error = class extends Error { + + constructor(message) { + super(message); + this.name = 'Error loading ESP-DL model.'; + } +}; + +export const ModelFactory = espdl.ModelFactory; \ No newline at end of file diff --git a/source/view.js b/source/view.js index c6c15e480d..846bba2258 100644 --- a/source/view.js +++ b/source/view.js @@ -6301,6 +6301,7 @@ view.ModelFactoryService = class { this.register('./onednn', ['.json']); this.register('./espresso', ['.espresso.net', '.espresso.shape', '.espresso.weights'], ['.mlmodelc']); this.register('./mlir', ['.mlir', '.mlir.txt', '.mlirbc', '.txt']); + this.register('./espdl', ['.espdl'], [], [/^EDL2/]); this.register('./sentencepiece', ['.model']); this.register('./hailo', ['.hn', '.har', '.metadata.json']); this.register('./tvm', ['.json', '.params']); @@ -6860,7 +6861,6 @@ view.ModelFactoryService = class { { name: 'CviModel data', value: /^CviModel/ }, // https://github.com/sophgo/tpu-mlir/blob/master/include/tpu_mlir/Builder/CV18xx/proto/cvimodel.fbs { name: 'DRTcrypt data', value: /^DRTcrypt/ }, { name: 'ELF executable', value: /^\x7FELF/ }, - { name: 'EDL2 data', value: /^EDL2/ }, { name: 'encrypted data', value: /^ENCRYPTED_FILE|EV_ENCRYPTED/ }, { name: 'encrypted data', value: /^Salted__/ }, { name: 'encrypted data', value: /^KINGSOFTOFFICE/ }, diff --git a/test/models.json b/test/models.json index 76323c8ec8..7113925899 100644 --- a/test/models.json +++ b/test/models.json @@ -8809,5 +8809,26 @@ "source": "https://github.com/lutzroeder/netron/files/7083633/pt_face-quality_80_60_61.68M_1.4.zip[pt_face-quality_80_60_61.68M_1.4/quantized/PointsQuality_int.xmodel]", "format": "xmodel", "link": "https://github.com/lutzroeder/netron/issues/718" + }, + { + "type": "espdl", + "target": "imagenet_cls_mobilenetv2_s8_v1.espdl", + "source": "https://raw.githubusercontent.com/espressif/esp-dl/master/models/imagenet_cls/models/p4/imagenet_cls_mobilenetv2_s8_v1.espdl", + "format": "espdl", + "link": "https://github.com/espressif/esp-dl/blob/master/models/imagenet_cls/models/p4/imagenet_cls_mobilenetv2_s8_v1.espdl" + }, + { + "type": "espdl", + "target": "coco_detect_yolo11n_s8_v3.espdl", + "source": "https://raw.githubusercontent.com/espressif/esp-dl/master/models/coco_detect/models/p4/coco_detect_yolo11n_s8_v3.espdl", + "format": "espdl", + "link": "https://github.com/espressif/esp-dl/blob/master/models/coco_detect/models/p4/coco_detect_yolo11n_s8_v3.espdl" + }, + { + "type": "espdl", + "target": "espdet_pico_416_416_cat.espdl", + "source": "https://raw.githubusercontent.com/espressif/esp-dl/master/models/cat_detect/models/p4/espdet_pico_416_416_cat.espdl", + "format": "espdl", + "link": "https://github.com/espressif/esp-dl/blob/master/models/cat_detect/models/p4/espdet_pico_416_416_cat.espdl" } -] \ No newline at end of file +] diff --git a/tools/espdl b/tools/espdl new file mode 100755 index 0000000000..61e384d9ab --- /dev/null +++ b/tools/espdl @@ -0,0 +1,47 @@ +#!/bin/bash + +set -e +pushd $(cd $(dirname ${0})/..; pwd) > /dev/null + +clean() { + echo "espdl clean" + rm -rf "./third_party/source/espdl" +} + +sync() { + echo "espdl sync" + mkdir -p "./third_party/source/espdl/metadata" + curl --silent --show-error --location --output "./third_party/source/espdl/metadata/espdl.fbs" "https://github.com/espressif/esp-dl/raw/refs/heads/master/esp-dl/fbs_loader/espdl.fbs" +} + + + +schema() { + echo "espdl schema" + [[ $(grep -U $'\x0D' ./source/espdl-schema.js) ]] && crlf=1 + node ./tools/flatc.js --root espdl --out ./source/espdl-schema.js ./third_party/source/espdl/metadata/espdl.fbs + # Fix ESLint errors for AlignedBytes.decode + sed -i '/espdl\.AlignedBytes = class AlignedBytes {/,/^};/ s/static decode(reader, position) {/static decode() {/' ./source/espdl-schema.js + if [[ -n ${crlf} ]]; then + unix2dos --quiet --newfile ./source/espdl-schema.js ./source/espdl-schema.js + fi +} + +metadata() { + echo "espdl metadata" + if [[ $(grep -U $'\x0D' ./source/espdl-metadata.json) ]]; then crlf=1; else crlf=; fi + node ./tools/espdl-script.js + if [[ -n ${crlf} ]]; then + unix2dos --quiet --newfile ./source/espdl-metadata.json ./source/espdl-metadata.json + fi +} + +while [ "$#" != 0 ]; do + command="$1" && shift + case "${command}" in + "clean") clean;; + "sync") sync;; + "schema") schema;; + "metadata") metadata;; + esac +done diff --git a/tools/espdl-script.js b/tools/espdl-script.js new file mode 100644 index 0000000000..001991e58c --- /dev/null +++ b/tools/espdl-script.js @@ -0,0 +1,110 @@ +import * as fs from 'fs/promises'; +import * as path from 'path'; +import * as url from 'url'; + +const main = async () => { + const dirname = path.dirname(url.fileURLToPath(import.meta.url)); + const file = path.join(dirname, '..', 'source', 'espdl-metadata.json'); + + // Read existing metadata + const input = await fs.readFile(file, 'utf-8'); + const json = JSON.parse(input); + + // Validate and process operators + const operators = new Map(); + const errors = []; + + for (const operator of json) { + // Check for duplicate operator names + if (operators.has(operator.name)) { + errors.push(`Duplicate operator '${operator.name}'.`); + } + operators.set(operator.name, operator); + + // Validate required fields + if (!operator.name) { + errors.push(`Operator missing 'name' field.`); + } + if (!operator.module) { + errors.push(`Operator '${operator.name}' missing 'module' field.`); + } + if (operator.version === undefined) { + errors.push(`Operator '${operator.name}' missing 'version' field.`); + } + if (!operator.description) { + errors.push(`Operator '${operator.name}' missing 'description' field.`); + } + + // Validate inputs + if (operator.inputs && Array.isArray(operator.inputs)) { + const inputNames = new Set(); + for (const input of operator.inputs) { + if (!input.name) { + errors.push(`Operator '${operator.name}' has input missing 'name' field.`); + } + if (inputNames.has(input.name)) { + errors.push(`Operator '${operator.name}' has duplicate input name '${input.name}'.`); + } + inputNames.add(input.name); + } + } + + // Validate outputs + if (operator.outputs && Array.isArray(operator.outputs)) { + const outputNames = new Set(); + for (const output of operator.outputs) { + if (!output.name) { + errors.push(`Operator '${operator.name}' has output missing 'name' field.`); + } + if (outputNames.has(output.name)) { + errors.push(`Operator '${operator.name}' has duplicate output name '${output.name}'.`); + } + outputNames.add(output.name); + } + } + } + + // Report any errors + if (errors.length > 0) { + throw new Error(`ESPDL metadata validation errors:\n${errors.join('\n')}`); + } + + // Sort operators by name + const beforeSort = json.map((op) => op.name); + json.sort((a, b) => a.name.localeCompare(b.name)); + const afterSort = json.map((op) => op.name); + + // Check if sorting changed anything + let changed = false; + for (let i = 0; i < beforeSort.length; i++) { + if (beforeSort[i] !== afterSort[i]) { + changed = true; + break; + } + } + + if (changed) { + // eslint-disable-next-line no-console + console.log(`Sorted ${json.length} operators.`); + } + + // Format JSON with consistent indentation + let output = JSON.stringify(json, null, 2); + + // Apply formatting similar to other metadata files + output = output.replace(/\s {8}/g, ' '); + output = output.replace(/,\s {8}/g, ', '); + output = output.replace(/\s {6}}/g, ' }'); + + // Write back to file + await fs.writeFile(file, output, 'utf-8'); + + // eslint-disable-next-line no-console + console.log(`Processed ${json.length} ESPDL operators.`); +}; + +await main().catch((error) => { + // eslint-disable-next-line no-console + console.error(error.message); + process.exit(1); +}); \ No newline at end of file