Skip to content

Commit

Permalink
Respond to PR feedback: fix typos and outline methods.
Browse files Browse the repository at this point in the history
  • Loading branch information
nicktobey committed Jan 6, 2025
1 parent d844f69 commit 84f0b04
Show file tree
Hide file tree
Showing 4 changed files with 48 additions and 38 deletions.
2 changes: 1 addition & 1 deletion go/libraries/doltcore/sqle/tables.go
Original file line number Diff line number Diff line change
Expand Up @@ -2901,7 +2901,7 @@ func (t *WritableDoltTable) UpdateForeignKey(ctx *sql.Context, fkName string, sq
// CreateIndexForForeignKey implements sql.ForeignKeyTable
func (t *AlterableDoltTable) CreateIndexForForeignKey(ctx *sql.Context, idx sql.IndexDef) error {
if idx.Constraint != sql.IndexConstraint_None && idx.Constraint != sql.IndexConstraint_Unique && idx.Constraint != sql.IndexConstraint_Spatial {
return fmt.Errorf("only the following types of index constraints are supported for foriegn keys: none, unique, spatial")
return fmt.Errorf("only the following types of index constraints are supported for foreign keys: none, unique, spatial")
}
columns := make([]string, len(idx.Columns))
for i, indexCol := range idx.Columns {
Expand Down
72 changes: 43 additions & 29 deletions go/libraries/doltcore/table/editor/creation/external_build_index.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,35 +59,7 @@ func BuildProllyIndexExternal(ctx *sql.Context, vrw types.ValueReadWriter, ns tr
}

if idx.IsVector() {
// Secondary indexes are always covering and have no non-key columns
valDesc := val.NewTupleDescriptor()
proximityMapBuilder, err := prolly.NewProximityMapBuilder(ctx, ns, idx.VectorProperties().DistanceType, keyDesc, valDesc, prolly.DefaultLogChunkSize)
if err != nil {
return nil, err
}
for {
k, v, err := iter.Next(ctx)
if err == io.EOF {
break
} else if err != nil {
return nil, err
}

idxKey, err := secondaryBld.SecondaryKeyFromRow(ctx, k, v)
if err != nil {
return nil, err
}

if uniqCb != nil && prefixDesc.HasNulls(idxKey) {
continue
}

if err := proximityMapBuilder.Insert(ctx, idxKey, val.EmptyTuple); err != nil {
return nil, err
}
}
proximityMap, err := proximityMapBuilder.Flush(ctx)
return durable.IndexFromProximityMap(proximityMap), nil
return BuildProximityIndex(ctx, ns, idx, keyDesc, prefixDesc, iter, secondaryBld, uniqCb)
}

sorter := sort.NewTupleSorter(batchSize, fileMax, func(t1, t2 val.Tuple) bool {
Expand Down Expand Up @@ -144,6 +116,48 @@ func BuildProllyIndexExternal(ctx *sql.Context, vrw types.ValueReadWriter, ns tr
return durable.IndexFromProllyMap(ret), nil
}

// func BuildProximityIndexExternal(ctx *sql.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, tableName string, idx schema.Index, primary prolly.Map, uniqCb DupEntryCb) (durable.Index, error) {
func BuildProximityIndex(
ctx *sql.Context,
ns tree.NodeStore,
idx schema.Index,
keyDesc val.TupleDesc,
prefixDesc val.TupleDesc,
iter prolly.MapIter,
secondaryBld index.SecondaryKeyBuilder,
uniqCb DupEntryCb,
) (durable.Index, error) {
// Secondary indexes have no non-key columns
valDesc := val.NewTupleDescriptor()
proximityMapBuilder, err := prolly.NewProximityMapBuilder(ctx, ns, idx.VectorProperties().DistanceType, keyDesc, valDesc, prolly.DefaultLogChunkSize)
if err != nil {
return nil, err
}
for {
k, v, err := iter.Next(ctx)
if err == io.EOF {
break
} else if err != nil {
return nil, err
}

idxKey, err := secondaryBld.SecondaryKeyFromRow(ctx, k, v)
if err != nil {
return nil, err
}

if uniqCb != nil && prefixDesc.HasNulls(idxKey) {
continue
}

if err := proximityMapBuilder.Insert(ctx, idxKey, val.EmptyTuple); err != nil {
return nil, err
}
}
proximityMap, err := proximityMapBuilder.Flush(ctx)
return durable.IndexFromProximityMap(proximityMap), nil
}

type tupleIterWithCb struct {
iter sort.KeyIter
err error
Expand Down
2 changes: 1 addition & 1 deletion go/serial/schema.fbs
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ table Index {
fulltext_key:bool;
fulltext_info:FulltextInfo;

// fulltext information
// vector information
// these fields should be set for vector indexes and otherwise omitted, for backwards compatibility
vector_key:bool;
vector_info:VectorInfo;
Expand Down
10 changes: 3 additions & 7 deletions go/serial/vectorindexnode.fbs
Original file line number Diff line number Diff line change
Expand Up @@ -19,25 +19,21 @@ namespace serial;
table VectorIndexNode {
// sorted array of key items
key_items:[ubyte] (required);
// items offets for |key_items|
// item offsets for |key_items|
// first offset is 0, last offset is len(key_items)
key_offsets:[uint32] (required);
// item type for |key_items|
// key_type:ItemType;

// array of values items, ordered by paired key
value_items:[ubyte];
// item offsets for |value_items|
// first offset is 0, last offset is len(value_items)
value_offsets:[uint32];
// item type for |value_items|
// value_type:ItemType;

// array of chunk addresses
// - subtree addresses for internal prolly tree nodes
// - value addresses for AddressMap leaf nodes
// node that while the keys in this index are addresses to JSON chunks, we don't store those in the address_array
// because we are guarenteed to have other references to those chunks in the primary index.
// note that while the keys in this index are addresses to JSON chunks, we don't store those in the address_array
// because we are guaranteed to have other references to those chunks in the primary index.
address_array:[ubyte] (required);

// array of varint encoded subtree counts
Expand Down

0 comments on commit 84f0b04

Please sign in to comment.