Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

MDEV-33408 Introduce session variables to manage HNSW index parameters #3226

Open
wants to merge 28 commits into
base: bb-11.4-vec-vicentiu-hugo
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
7b87e01
MDEV-32885 VEC_DISTANCE() function
vuvova Nov 25, 2023
ed8b6da
make INFORMATION_SCHEMA.STATISTICS.COMMENT not nullable
vuvova Jan 19, 2024
4bea056
fix main.plugin_vars test to cleanup after itself
vuvova Feb 5, 2024
62e6031
cleanup: spaces, casts, comments
vuvova Jan 8, 2024
e409868
cleanup: pass TABLE_SHARE to store_key_options()
vuvova Jan 26, 2024
bc8a7a3
reject invalid spatial key declarations in the parser
vuvova Jan 8, 2024
16846b5
cleanup: remove unconditional #ifdef's
vuvova Jan 10, 2024
2cd2320
cleanup: lex_string_set3()
vuvova Jan 27, 2024
8ddb99a
cleanup: Queue and Bounded_queue
vuvova Feb 6, 2024
ef14f42
cleanup: key algorithm vs key flags
vuvova Jan 14, 2024
022bc34
cleanup: make_long_hash_field_name() and add_hash_field()
vuvova Jan 18, 2024
38e84a8
cleanup: generalize ER_SPATIAL_CANT_HAVE_NULL
vuvova Jan 17, 2024
f1d0352
cleanup: generalize ER_INNODB_NO_FT_TEMP_TABLE
vuvova Jan 25, 2024
0cbd050
cleanup: extract ha_create_table_from_share()
vuvova Jan 25, 2024
cdcf739
open frm for DROP TABLE
vuvova Jan 26, 2024
3fa8be1
cleanup: unused function argument
vuvova Jan 26, 2024
cc82b35
Revert "MDEV-15458 Segfault in heap_scan() upon UPDATE after ADD SYST…
vuvova Feb 9, 2024
a857b68
initial support for vector indexes
vuvova Jan 17, 2024
9ca6554
Initial fixup
cvicentiu Feb 17, 2024
1568677
Wip
cvicentiu Feb 21, 2024
e944876
Graph insert possibly working ok
cvicentiu Feb 22, 2024
b34a896
Search is now working, but layer unaware
cvicentiu Feb 22, 2024
8aa7c1e
wip
cvicentiu Feb 22, 2024
3d0e4ea
Vec insert and search working on a multi-layer
cvicentiu Feb 23, 2024
437e214
MDEV-33408 Alter HNSW graph storage and fix memory leak
HugoWenTD Apr 12, 2024
ee2cc47
Support files for ann-workspace
cvicentiu Apr 15, 2024
7829259
Bug fixes - on top of Hugo's patch
cvicentiu Apr 18, 2024
5899540
Introduce session variables to manage HNSW index parameters
HugoWenTD Apr 26, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Prev Previous commit
Next Next commit
cleanup: spaces, casts, comments
  • Loading branch information
vuvova authored and cvicentiu committed Apr 4, 2024
commit 62e603162409bf4d301f10b2e007ed735299de18
16 changes: 8 additions & 8 deletions include/my_base.h
Original file line number Diff line number Diff line change
Expand Up @@ -97,16 +97,16 @@ enum ha_rkey_function {
HA_READ_MBR_EQUAL
};

/* Key algorithm types */
/* Key algorithm types (stored in .frm) */

enum ha_key_alg {
HA_KEY_ALG_UNDEF= 0, /* Not specified (old file) */
HA_KEY_ALG_BTREE= 1, /* B-tree, default one */
HA_KEY_ALG_RTREE= 2, /* R-tree, for spatial searches */
HA_KEY_ALG_HASH= 3, /* HASH keys (HEAP tables) */
HA_KEY_ALG_FULLTEXT= 4, /* FULLTEXT (MyISAM tables) */
HA_KEY_ALG_LONG_HASH= 5, /* long BLOB keys */
HA_KEY_ALG_UNIQUE_HASH= 6 /* Internal UNIQUE hash (Aria) */
HA_KEY_ALG_UNDEF= 0, /* Not specified (old file) */
HA_KEY_ALG_BTREE= 1, /* B-tree, default one */
HA_KEY_ALG_RTREE= 2, /* R-tree, for spatial searches */
HA_KEY_ALG_HASH= 3, /* HASH keys (HEAP tables) */
HA_KEY_ALG_FULLTEXT= 4, /* FULLTEXT */
HA_KEY_ALG_LONG_HASH= 5, /* long BLOB keys */
HA_KEY_ALG_UNIQUE_HASH= 6 /* Internal UNIQUE hash (Aria) */
};

/* Storage media types */
Expand Down
3 changes: 0 additions & 3 deletions mysql-test/main/myisam.result
Original file line number Diff line number Diff line change
Expand Up @@ -454,9 +454,6 @@ select * from t1 where a='807780' and b='477' and c='165';
a b c
807780 477 165
drop table t1;
DROP TABLE IF EXISTS t1;
Warnings:
Note 1051 Unknown table 'test.t1'
CREATE TABLE t1 (a varchar(150) NOT NULL, KEY (a));
INSERT t1 VALUES ("can \tcan");
INSERT t1 VALUES ("can can");
Expand Down
7 changes: 3 additions & 4 deletions mysql-test/main/myisam.test
Original file line number Diff line number Diff line change
Expand Up @@ -438,7 +438,6 @@ drop table t1;
#
# space-stripping in _mi_prefix_search: BUG#5284
#
DROP TABLE IF EXISTS t1;
CREATE TABLE t1 (a varchar(150) NOT NULL, KEY (a));
INSERT t1 VALUES ("can \tcan");
INSERT t1 VALUES ("can can");
Expand Down Expand Up @@ -1129,7 +1128,7 @@ drop table t1;
create table t1 (c1 int) engine=myisam pack_keys=0;
create table t2 (c1 int) engine=myisam pack_keys=1;
create table t3 (c1 int) engine=myisam pack_keys=default;
--error 1064
--error ER_PARSE_ERROR
create table t4 (c1 int) engine=myisam pack_keys=2;
drop table t1, t2, t3;

Expand Down Expand Up @@ -1316,9 +1315,9 @@ create table t1 (a int not null, key `a` (a) key_block_size=1025);
show create table t1;
drop table t1;

--error 1064
--error ER_PARSE_ERROR
create table t1 (a int not null, key key_block_size=1024 (a));
--error 1064
--error ER_PARSE_ERROR
create table t1 (a int not null, key `a` key_block_size=1024 (a));

#
Expand Down
2 changes: 0 additions & 2 deletions mysql-test/suite/maria/maria.result
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@ set global default_storage_engine=aria;
set session default_storage_engine=aria;
set global aria_page_checksum=0;
set global aria_log_file_size=4294959104;
drop table if exists t1,t2;
drop view if exists v1;
SET SQL_WARNINGS=1;
CREATE TABLE t1 (
STRING_DATA char(255) default NULL,
Expand Down
10 changes: 3 additions & 7 deletions mysql-test/suite/maria/maria.test
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,6 @@ let $default_log_file_size=`select @@global.aria_log_file_size`;
set global aria_log_file_size=4294959104;

# Initialise
--disable_warnings
drop table if exists t1,t2;
drop view if exists v1;
--enable_warnings
SET SQL_WARNINGS=1;

#
Expand Down Expand Up @@ -1061,7 +1057,7 @@ drop table t1;
create table t1 (c1 int) pack_keys=0;
create table t2 (c1 int) pack_keys=1;
create table t3 (c1 int) pack_keys=default;
--error 1064
--error ER_PARSE_ERROR
create table t4 (c1 int) pack_keys=2;
drop table t1, t2, t3;

Expand Down Expand Up @@ -1208,9 +1204,9 @@ create table t1 (a int not null, key `a` (a) key_block_size=1025);
show create table t1;
drop table t1;

--error 1064
--error ER_PARSE_ERROR
create table t1 (a int not null, key key_block_size=1024 (a));
--error 1064
--error ER_PARSE_ERROR
create table t1 (a int not null, key `a` key_block_size=1024 (a));

#
Expand Down
13 changes: 5 additions & 8 deletions sql/filesort.cc
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ static ha_rows find_all_keys(THD *thd, Sort_param *param, SQL_SELECT *select,
ha_rows *found_rows);
static bool write_keys(Sort_param *param, SORT_INFO *fs_info,
uint count, IO_CACHE *buffer_file, IO_CACHE *tempfile);
static uint make_sortkey(Sort_param *param, uchar *to, uchar *ref_pos,
static uint make_sortkey(Sort_param *, uchar *, uchar *,
bool using_packed_sortkeys= false);
static uint make_sortkey(Sort_param *param, uchar *to);
static uint make_packed_sortkey(Sort_param *param, uchar *to);
Expand Down Expand Up @@ -1510,8 +1510,7 @@ static void register_used_fields(Sort_param *param)
}


static bool save_index(Sort_param *param, uint count,
SORT_INFO *table_sort)
static bool save_index(Sort_param *param, uint count, SORT_INFO *table_sort)
{
uint offset,res_length, length;
uchar *to;
Expand Down Expand Up @@ -2945,16 +2944,14 @@ static uint make_sortkey(Sort_param *param, uchar *to)
{
bool maybe_null=0;
if ((field=sort_field->field))
{
// Field
{ // Field
field->make_sort_key_part(to, sort_field->length);
if ((maybe_null= field->maybe_null()))
to++;
}
else
{ // Item
sort_field->item->type_handler()->make_sort_key_part(to,
sort_field->item,
{ // Item
sort_field->item->type_handler()->make_sort_key_part(to, sort_field->item,
sort_field,
&param->tmp_buffer);
if ((maybe_null= sort_field->item->maybe_null()))
Expand Down
14 changes: 5 additions & 9 deletions sql/ha_partition.cc
Original file line number Diff line number Diff line change
Expand Up @@ -116,9 +116,8 @@ int ha_partition::notify_tabledef_changed(LEX_CSTRING *db,
{
LEX_CSTRING table_name;
const char *table_name_ptr;
if (create_partition_name(from_buff, sizeof(from_buff),
from_path, name_buffer_ptr,
NORMAL_PART_NAME, FALSE))
if (create_partition_name(from_buff, sizeof(from_buff), from_path,
name_buffer_ptr, NORMAL_PART_NAME, FALSE))
res=1;
table_name_ptr= from_buff + dirname_length(from_buff);

Expand All @@ -134,12 +133,9 @@ int ha_partition::notify_tabledef_changed(LEX_CSTRING *db,


static int
partition_notify_tabledef_changed(handlerton *,
LEX_CSTRING *db,
LEX_CSTRING *table,
LEX_CUSTRING *frm,
LEX_CUSTRING *version,
handler *file)
partition_notify_tabledef_changed(handlerton *, LEX_CSTRING *db,
LEX_CSTRING *table, LEX_CUSTRING *frm,
LEX_CUSTRING *version, handler *file)
{
DBUG_ENTER("partition_notify_tabledef_changed");
DBUG_RETURN(static_cast<ha_partition*>
Expand Down
15 changes: 15 additions & 0 deletions sql/handler.cc
Original file line number Diff line number Diff line change
Expand Up @@ -3192,6 +3192,13 @@ Lex_cstring handler::get_canonical_filename(const Lex_cstring &path,
/**
Delete a table in the engine

@param thd
@param hton
@param path no extension, e.g. "./test/t1"
@param db for the error message only
@param alias table name, for the error message only
@param generate_warning generate "table not found" warnings as needed

@return 0 Table was deleted
@return -1 Table didn't exists, no error given
@return # Error from table handler
Expand Down Expand Up @@ -6107,6 +6114,14 @@ int handler::calculate_checksum()
/**
Initiates table-file and calls appropriate database-creator.

@param thd
@param path no extension, e.g. "./test/t1"
@param db
@param table_name
@param create_info
@param frm an frm image or NULL (meaning, read it from the file)
@param skip_frm_file do not write the frm image to the .frm file

@retval
0 ok
@retval
Expand Down
2 changes: 1 addition & 1 deletion sql/handler.h
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ enum chf_create_flags {
*/
#define HA_REQUIRES_KEY_COLUMNS_FOR_DELETE (1ULL << 6)
#define HA_NULL_IN_KEY (1ULL << 7) /* One can have keys with NULL */
#define HA_DUPLICATE_POS (1ULL << 8) /* ha_position() gives dup row */
#define HA_DUPLICATE_POS (1ULL << 8) /* position() gives dup row */
#define HA_NO_BLOBS (1ULL << 9) /* Doesn't support blobs */
#define HA_CAN_INDEX_BLOBS (1ULL << 10)
#define HA_AUTO_PART_KEY (1ULL << 11) /* auto-increment in multi-part key */
Expand Down
2 changes: 1 addition & 1 deletion sql/item.h
Original file line number Diff line number Diff line change
Expand Up @@ -2750,7 +2750,7 @@ inline Item* get_item_copy (THD *thd, T* item)
if (likely(copy))
copy->register_in(thd);
return copy;
}
}


#ifndef DBUG_OFF
Expand Down
24 changes: 7 additions & 17 deletions sql/sql_select.cc
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,6 @@ static int join_init_quick_read_record(JOIN_TAB *tab);
static quick_select_return test_if_quick_select(JOIN_TAB *tab);
static int test_if_use_dynamic_range_scan(JOIN_TAB *join_tab);
static int join_read_first(JOIN_TAB *tab);
static int join_read_next(READ_RECORD *info);
static int join_read_next_same(READ_RECORD *info);
static int join_read_last(JOIN_TAB *tab);
static int join_read_prev_same(READ_RECORD *info);
Expand Down Expand Up @@ -7188,8 +7187,7 @@ add_keyuse(DYNAMIC_ARRAY *keyuse_array, KEY_FIELD *key_field,

static LEX_CSTRING equal_str= { STRING_WITH_LEN("=") };

static bool
add_key_part(DYNAMIC_ARRAY *keyuse_array, KEY_FIELD *key_field)
static bool add_key_part(DYNAMIC_ARRAY *keyuse_array, KEY_FIELD *key_field)
{
Field *field=key_field->field;
TABLE *form= field->table;
Expand All @@ -7202,7 +7200,7 @@ add_key_part(DYNAMIC_ARRAY *keyuse_array, KEY_FIELD *key_field)
if (!(form->keys_in_use_for_query.is_set(key)))
continue;
if (form->key_info[key].flags & (HA_FULLTEXT | HA_SPATIAL))
continue; // ToDo: ft-keys in non-ft queries. SerG
continue;

KEY *keyinfo= form->key_info+key;
uint key_parts= form->actual_n_key_parts(keyinfo);
Expand Down Expand Up @@ -25983,7 +25981,6 @@ part_of_refkey(TABLE *table,Field *field)
@param used_key_parts [out] NULL by default, otherwise return value for
used key parts.


@note
used_key_parts is set to correct key parts used if return value != 0
(On other cases, used_key_part may be changed)
Expand All @@ -25999,9 +25996,8 @@ part_of_refkey(TABLE *table,Field *field)
-1 Reverse key can be used
*/

static int test_if_order_by_key(JOIN *join,
ORDER *order, TABLE *table, uint idx,
uint *used_key_parts)
static int test_if_order_by_key(JOIN *join, ORDER *order, TABLE *table,
uint idx, uint *used_key_parts)
{
KEY_PART_INFO *key_part,*key_part_end;
key_part=table->key_info[idx].key_part;
Expand All @@ -26016,8 +26012,7 @@ static int test_if_order_by_key(JOIN *join,
DBUG_ENTER("test_if_order_by_key");

if ((table->file->ha_table_flags() & HA_PRIMARY_KEY_IN_READ_INDEX) &&
table->key_info[idx].ext_key_part_map &&
pk != MAX_KEY && pk != idx)
table->key_info[idx].ext_key_part_map && pk != MAX_KEY && pk != idx)
{
have_pk_suffix= true;
}
Expand Down Expand Up @@ -26485,10 +26480,7 @@ test_if_skip_sort_order(JOIN_TAB *tab,ORDER *order,ha_rows select_limit,
{
Item *item= (*tmp_order->item)->real_item();
if (item->type() != Item::FIELD_ITEM)
{
usable_keys.clear_all();
DBUG_RETURN(0);
}

/*
Take multiple-equalities into account. Suppose we have
Expand Down Expand Up @@ -32071,8 +32063,7 @@ test_if_cheaper_ordering(const JOIN_TAB *tab, ORDER *order, TABLE *table,
temporary table + filesort could be cheaper for grouping
queries too.
*/
if (is_covering ||
has_limit ||
if (is_covering || has_limit ||
(ref_key < 0 && (group || table->force_index)))
{
double rec_per_key;
Expand Down Expand Up @@ -32335,8 +32326,7 @@ uint get_index_for_order(ORDER *order, TABLE *table, SQL_SELECT *select,

int key, direction;
if (test_if_cheaper_ordering(NULL, order, table,
table->keys_in_use_for_order_by, -1,
limit,
table->keys_in_use_for_order_by, -1, limit,
&key, &direction, &limit) &&
!is_key_used(table, key, table->write_set))
{
Expand Down
30 changes: 12 additions & 18 deletions sql/sql_show.cc
Original file line number Diff line number Diff line change
Expand Up @@ -2523,7 +2523,6 @@ static void store_key_options(THD *thd, String *packet, TABLE *table,
MODE_DB2 |
MODE_MAXDB |
MODE_ANSI)) != 0;
char *end, buff[32];

if (!(thd->variables.sql_mode & MODE_NO_KEY_OPTIONS) &&
!limited_mysql_mode && !foreign_db_mode)
Expand All @@ -2545,8 +2544,7 @@ static void store_key_options(THD *thd, String *packet, TABLE *table,
table->s->key_block_size != key_info->block_size)
{
packet->append(STRING_WITH_LEN(" KEY_BLOCK_SIZE="));
end= longlong10_to_str(key_info->block_size, buff, 10);
packet->append(buff, (uint) (end - buff));
packet->append_ulonglong(key_info->block_size);
}
DBUG_ASSERT(MY_TEST(key_info->flags & HA_USES_COMMENT) ==
(key_info->comment.length > 0));
Expand Down Expand Up @@ -7144,17 +7142,16 @@ int fill_schema_proc(THD *thd, TABLE_LIST *tables, COND *cond)
}


static int get_schema_stat_record(THD *thd, TABLE_LIST *tables,
TABLE *table, bool res,
const LEX_CSTRING *db_name,
static int get_schema_stat_record(THD *thd, TABLE_LIST *tables, TABLE *table,
bool res, const LEX_CSTRING *db_name,
const LEX_CSTRING *table_name)
{
CHARSET_INFO *cs= system_charset_info;
DBUG_ENTER("get_schema_stat_record");
if (!tables->view)
{
TABLE *show_table= tables->table;
KEY *key_info=show_table->s->key_info;
KEY *key_info= show_table->s->key_info;
if (show_table->file)
{
(void) read_statistics_for_tables(thd, tables, false);
Expand Down Expand Up @@ -7208,21 +7205,18 @@ static int get_schema_stat_record(THD *thd, TABLE_LIST *tables,
table->field[0]->store(STRING_WITH_LEN("def"), cs);
table->field[1]->store(db_name->str, db_name->length, cs);
table->field[2]->store(table_name->str, table_name->length, cs);
table->field[3]->store((longlong) ((key_info->flags &
HA_NOSAME) ? 0 : 1), TRUE);
table->field[3]->store(key_info->flags & HA_NOSAME ? 0 : 1, TRUE);
table->field[4]->store(db_name->str, db_name->length, cs);
table->field[5]->store(key_info->name.str, key_info->name.length, cs);
table->field[6]->store((longlong) (j+1), TRUE);
str= (key_part->field ? &key_part->field->field_name :
&unknown);
str= key_part->field ? &key_part->field->field_name : &unknown;
table->field[7]->store(str->str, str->length, cs);
if (show_table->file)
{
if (show_table->file->index_flags(i, j, 0) & HA_READ_ORDER)
{
table->field[8]->store(((key_part->key_part_flag &
HA_REVERSE_SORT) ?
"D" : "A"), 1, cs);
table->field[8]->store(key_part->key_part_flag & HA_REVERSE_SORT
? "D" : "A", 1, cs);
table->field[8]->set_notnull();
}
if (key_info->algorithm == HA_KEY_ALG_LONG_HASH)
Expand All @@ -7234,11 +7228,11 @@ static int get_schema_stat_record(THD *thd, TABLE_LIST *tables,
from table as key_info points to TABLE_SHARE which has no
statistics.
*/
KEY *key_info= show_table->key_info + i;
if (key_info->rec_per_key[j])
KEY *keyinfo= show_table->key_info + i;
if (keyinfo->rec_per_key[j])
{
ha_rows records= (ha_rows) ((double) show_table->stat_records() /
key_info->actual_rec_per_key(j));
keyinfo->actual_rec_per_key(j));
table->field[9]->store((longlong) records, TRUE);
table->field[9]->set_notnull();
}
Expand All @@ -7256,7 +7250,7 @@ static int get_schema_stat_record(THD *thd, TABLE_LIST *tables,
table->field[10]->set_notnull();
}
uint flags= key_part->field ? key_part->field->flags : 0;
const char *pos=(char*) ((flags & NOT_NULL_FLAG) ? "" : "YES");
const char *pos= flags & NOT_NULL_FLAG ? "" : "YES";
table->field[12]->store(pos, strlen(pos), cs);
if (!show_table->s->keys_in_use.is_set(i))
table->field[14]->store(STRING_WITH_LEN("disabled"), cs);
Expand Down