Skip to content

Commit efd87ae

Browse files
renecannaorenecannao
authored
Merge pull request #5254 from sysown/v3.0_pgsql-query-digest-gen-5253
Add PostgreSQL-Specific Tokenizer for Query Digest Generation
2 parents 2667540 + 0f7ff1f commit efd87ae

16 files changed

+5641
-18
lines changed

‎include/c_tokenizer.h‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -40,17 +40,25 @@ typedef struct _options {
4040
#ifdef __cplusplus
4141
extern "C" {
4242
#endif /* __cplusplus */
43-
//tokenizer_t tokenizer( const char* s, const char* delimiters, int empties );
4443
void tokenizer( tokenizer_t *, const char* s, const char* delimiters, int empties );
4544
const char* free_tokenizer( tokenizer_t* tokenizer );
4645
const char* tokenize( tokenizer_t* tokenizer );
47-
char * mysql_query_digest_first_stage(const char* const q, int q_len, char** const fst_cmnt, char* const buf);
48-
char * mysql_query_digest_second_stage(const char* const q, int q_len, char** const fst_cmnt, char* const buf);
49-
char * mysql_query_digest_and_first_comment_2(const char* const q, int q_len, char** const fst_cmnt, char* const buf);
50-
char * mysql_query_digest_and_first_comment_one_it(char *s , int len , char **first_comment, char *buf);
46+
5147
void c_split_2(const char *in, const char *del, char **out1, char **out2);
52-
char * query_strip_comments(char* s, int len, bool lowercase);
53-
char * query_digest_and_first_comment_2(const char* const q, int q_len, char** const fst_cmnt, char* const buf, const options* opts);
48+
char* mysql_query_strip_comments(char* s, int len, bool lowercase);
49+
char* mysql_query_digest_and_first_comment(const char* const q, int q_len, char** const fst_cmnt, char* const buf, const options* opts);
50+
char* pgsql_query_strip_comments(char* s, int len, bool lowercase);
51+
char* pgsql_query_digest_and_first_comment(const char* const q, int q_len, char** const fst_cmnt, char* const buf, const options* opts);
52+
53+
// For TAP Test
54+
char* mysql_query_digest_first_stage(const char* const q, int q_len, char** const fst_cmnt, char* const buf);
55+
char* mysql_query_digest_second_stage(const char* const q, int q_len, char** const fst_cmnt, char* const buf);
56+
char* mysql_query_digest_and_first_comment_2(const char* const q, int q_len, char** const fst_cmnt, char* const buf);
57+
char* mysql_query_digest_and_first_comment_one_it(char* s, int len, char** first_comment, char* buf);
58+
char* pgsql_query_digest_first_stage(const char* const q, int q_len, char** const fst_cmnt, char* const buf);
59+
char* pgsql_query_digest_second_stage(const char* const q, int q_len, char** const fst_cmnt, char* const buf);
60+
char* pgsql_query_digest_and_first_comment_2(const char* const q, int q_len, char** const fst_cmnt, char* const buf);
61+
char* pgsql_query_digest_and_first_comment_one_it(char* s, int len, char** first_comment, char* buf);
5462
#ifdef __cplusplus
5563
}
5664
#endif /* __cplusplus */

‎lib/Makefile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,8 @@ _OBJ_CXX := ProxySQL_GloVars.oo network.oo debug.oo configfile.oo Query_Cache.oo
7777
MySQL_Query_Cache.oo PgSQL_Query_Cache.oo PgSQL_Monitor.oo \
7878
MySQL_Set_Stmt_Parser.oo PgSQL_Set_Stmt_Parser.oo \
7979
PgSQL_Variables_Validator.oo PgSQL_ExplicitTxnStateMgr.oo \
80-
PgSQL_PreparedStatement.oo PgSQL_Extended_Query_Message.oo
80+
PgSQL_PreparedStatement.oo PgSQL_Extended_Query_Message.oo \
81+
pgsql_tokenizer.oo
8182

8283
OBJ_CXX := $(patsubst %,$(ODIR)/%,$(_OBJ_CXX))
8384
HEADERS := ../include/*.h ../include/*.hpp

‎lib/MySQL_Monitor.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -260,8 +260,8 @@ class MySQL_Monitor_Connection_Pool {
260260
void conn_register(MySQL_Monitor_State_Data *mmsd) {
261261
#ifdef DEBUG
262262
std::lock_guard<std::mutex> lock(mutex);
263-
MYSQL *my = mmsd->mysql;
264263
pthread_mutex_lock(&m2);
264+
MYSQL* my = mmsd->mysql;
265265
__conn_register_label:
266266
for (unsigned int i=0; i<conns->len; i++) {
267267
MYSQL *my1 = (MYSQL *)conns->index(i);

‎lib/MySQL_Session.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7902,7 +7902,7 @@ bool MySQL_Session::handle_command_query_kill(PtrSize_t *pkt) {
79027902
MySQL_Connection *mc = client_myds->myconn;
79037903
if (mc->userinfo && mc->userinfo->username) {
79047904
if (CurrentQuery.MyComQueryCmd == MYSQL_COM_QUERY_KILL) {
7905-
char* qu = query_strip_comments((char *)pkt->ptr+1+sizeof(mysql_hdr), pkt->size-1-sizeof(mysql_hdr),
7905+
char* qu = mysql_query_strip_comments((char *)pkt->ptr+1+sizeof(mysql_hdr), pkt->size-1-sizeof(mysql_hdr),
79067906
mysql_thread___query_digests_lowercase);
79077907
string nq=string(qu,strlen(qu));
79087908
re2::RE2::Options *opt2=new re2::RE2::Options(RE2::Quiet);

‎lib/PgSQL_Session.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5191,7 +5191,7 @@ bool PgSQL_Session::handle_command_query_kill(PtrSize_t* pkt) {
51915191
if (mc->userinfo && mc->userinfo->username) {
51925192
if (CurrentQuery.PgQueryCmd == PGSQL_QUERY_CANCEL_BACKEND ||
51935193
CurrentQuery.PgQueryCmd == PGSQL_QUERY_TERMINATE_BACKEND) {
5194-
char* qu = query_strip_comments((char*)CurrentQuery.QueryPointer, CurrentQuery.QueryLength,
5194+
char* qu = pgsql_query_strip_comments((char*)CurrentQuery.QueryPointer, CurrentQuery.QueryLength,
51955195
pgsql_thread___query_digests_lowercase);
51965196
string nq = string(qu, strlen(qu));
51975197
re2::RE2::Options* opt2 = new re2::RE2::Options(RE2::Quiet);

‎lib/Query_Processor.cpp‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1820,8 +1820,13 @@ void Query_Processor<QP_DERIVED>::query_parser_init(SQP_par_t *qp, const char *q
18201820
opts.keep_comment = GET_THREAD_VARIABLE(query_digests_keep_comment);
18211821
opts.max_query_length = GET_THREAD_VARIABLE(query_digests_max_query_length);
18221822

1823-
qp->digest_text=query_digest_and_first_comment_2(query, query_length, &qp->first_comment,
1824-
((query_length < QUERY_DIGEST_BUF) ? qp->buf : NULL), &opts);
1823+
if constexpr (std::is_same_v<QP_DERIVED, MySQL_Query_Processor>) {
1824+
qp->digest_text = mysql_query_digest_and_first_comment(query, query_length, &qp->first_comment,
1825+
((query_length < QUERY_DIGEST_BUF) ? qp->buf : NULL), &opts);
1826+
} else if constexpr (std::is_same_v<QP_DERIVED, PgSQL_Query_Processor>) {
1827+
qp->digest_text = pgsql_query_digest_and_first_comment(query, query_length, &qp->first_comment,
1828+
((query_length < QUERY_DIGEST_BUF) ? qp->buf : NULL), &opts);
1829+
}
18251830
// the hash is computed only up to query_digests_max_digest_length bytes
18261831
const int digest_text_length=strnlen(qp->digest_text, GET_THREAD_VARIABLE(query_digests_max_digest_length));
18271832
qp->digest=SpookyHash::Hash64(qp->digest_text, digest_text_length, 0);

‎lib/c_tokenizer.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -519,7 +519,7 @@ void copy_next_char(shared_st* shared_st, const options* opts) {
519519
inc_proc_pos(shared_st);
520520
}
521521

522-
char cur_cmd_cmnt[FIRST_COMMENT_MAX_LENGTH];
522+
static char cur_cmd_cmnt[FIRST_COMMENT_MAX_LENGTH];
523523

524524
/**
525525
* @brief Safer version of 'is_digit_string' performing boundary checks.
@@ -2009,7 +2009,7 @@ char* mysql_query_digest_second_stage(const char* const q, int q_len, char** con
20092009
*
20102010
* @return A pointer to the start of the supplied buffer, or the allocated memory containing the digest.
20112011
*/
2012-
char* query_digest_and_first_comment_2(const char* const q, int q_len, char** const fst_cmnt, char* const buf, const options* opts) {
2012+
char* mysql_query_digest_and_first_comment(const char* const q, int q_len, char** const fst_cmnt, char* const buf, const options* opts) {
20132013
#ifdef DEBUG
20142014
if (buf != NULL) {
20152015
memset(buf, 0, 127);
@@ -2091,7 +2091,7 @@ char* mysql_query_digest_and_first_comment_2(const char* const q, int q_len, cha
20912091
// global options
20922092
options opts;
20932093
get_mysql_options(&opts);
2094-
return query_digest_and_first_comment_2(q, q_len, fst_cmnt, buf, &opts);
2094+
return mysql_query_digest_and_first_comment(q, q_len, fst_cmnt, buf, &opts);
20952095
}
20962096

20972097
static __attribute__((always_inline)) inline
@@ -2523,7 +2523,7 @@ char* mysql_query_digest_and_first_comment_one_it(char* q, int q_len, char** fst
25232523
return res;
25242524
}
25252525

2526-
char *query_strip_comments(char *s, int _len, bool lowercase) {
2526+
char* mysql_query_strip_comments(char *s, int _len, bool lowercase) {
25272527
int i = 0;
25282528
int len = _len;
25292529
char *r = (char *) malloc(len + SIZECHAR);

0 commit comments

Comments
 (0)