Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 21 additions & 18 deletions include/libs/function/functionMgt.h
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,6 @@ typedef enum EFunctionType {
FUNCTION_TYPE_DTW_PATH,
FUNCTION_TYPE_TLCC,


// math function
FUNCTION_TYPE_ABS = 1000,
FUNCTION_TYPE_LOG,
Expand Down Expand Up @@ -126,7 +125,10 @@ typedef enum EFunctionType {
FUNCTION_TYPE_FIND_IN_SET,
FUNCTION_TYPE_LIKE_IN_SET,
FUNCTION_TYPE_REGEXP_IN_SET,

FUNCTION_TYPE_FROM_BASE64,
FUNCTION_TYPE_SHA1,
FUNCTION_TYPE_SHA2,
FUNCTION_TYPE_MASK_FULL,

// conversion function
FUNCTION_TYPE_CAST = 2000,
Expand Down Expand Up @@ -178,19 +180,19 @@ typedef enum EFunctionType {
FUNCTION_TYPE_IROWTS_ORIGIN,
FUNCTION_TYPE_GROUP_ID,
FUNCTION_TYPE_IS_WINDOW_FILLED,
FUNCTION_TYPE_TPREV_TS, // _tprev_ts
FUNCTION_TYPE_TCURRENT_TS, // _tcurrent_ts
FUNCTION_TYPE_TNEXT_TS, // _tnext_ts
FUNCTION_TYPE_TWSTART, // _twstart
FUNCTION_TYPE_TWEND, // _twend
FUNCTION_TYPE_TWDURATION, // _twduration
FUNCTION_TYPE_TWROWNUM, // _twrownum
FUNCTION_TYPE_TPREV_LOCALTIME, // _tprev_localtime
FUNCTION_TYPE_TNEXT_LOCALTIME, // _tnext_localtime
FUNCTION_TYPE_TLOCALTIME, // _tlocaltime
FUNCTION_TYPE_TGRPID, // _tgrpid
FUNCTION_TYPE_PLACEHOLDER_COLUMN, // %%n
FUNCTION_TYPE_PLACEHOLDER_TBNAME, // %%tbname
FUNCTION_TYPE_TPREV_TS, // _tprev_ts
FUNCTION_TYPE_TCURRENT_TS, // _tcurrent_ts
FUNCTION_TYPE_TNEXT_TS, // _tnext_ts
FUNCTION_TYPE_TWSTART, // _twstart
FUNCTION_TYPE_TWEND, // _twend
FUNCTION_TYPE_TWDURATION, // _twduration
FUNCTION_TYPE_TWROWNUM, // _twrownum
FUNCTION_TYPE_TPREV_LOCALTIME, // _tprev_localtime
FUNCTION_TYPE_TNEXT_LOCALTIME, // _tnext_localtime
FUNCTION_TYPE_TLOCALTIME, // _tlocaltime
FUNCTION_TYPE_TGRPID, // _tgrpid
FUNCTION_TYPE_PLACEHOLDER_COLUMN, // %%n
FUNCTION_TYPE_PLACEHOLDER_TBNAME, // %%tbname
FUNCTION_TYPE_IMPUTATION_ROWTS,
FUNCTION_TYPE_IMPUTATION_MARK,
FUNCTION_TYPE_ANOMALY_MARK,
Expand Down Expand Up @@ -399,9 +401,10 @@ int32_t fmGetStreamPesudoFuncEnv(int32_t funcId, SNodeList* pParamNodes, SFuncEx

const void* fmGetStreamPesudoFuncVal(int32_t funcId, const SStreamRuntimeFuncInfo* pStreamRuntimeFuncInfo);

void fmGetStreamPesudoFuncValTbname(int32_t funcId, const SStreamRuntimeFuncInfo* pStreamRuntimeFuncInfo, void** data, int32_t* dataLen);
int32_t fmSetStreamPseudoFuncParamVal(int32_t funcId, SNodeList* pParamNodes, const SStreamRuntimeFuncInfo* pStreamRuntimeInfo);

void fmGetStreamPesudoFuncValTbname(int32_t funcId, const SStreamRuntimeFuncInfo* pStreamRuntimeFuncInfo, void** data,
int32_t* dataLen);
int32_t fmSetStreamPseudoFuncParamVal(int32_t funcId, SNodeList* pParamNodes,
const SStreamRuntimeFuncInfo* pStreamRuntimeInfo);

#ifdef __cplusplus
}
Expand Down
39 changes: 24 additions & 15 deletions include/libs/scalar/scalar.h
Original file line number Diff line number Diff line change
Expand Up @@ -38,18 +38,22 @@ int32_t scalarConvertOpValueNodeTs(SOperatorNode *node);
/*
pDst need to freed in caller
*/
int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst, const void* pExtraParam, void* streamTsRange);
int32_t scalarCalculateInRange(SNode *pNode, SArray *pBlockList, SScalarParam *pDst, int32_t rowStartIdx, int32_t rowEndIdx, const void* pExtraParam, void* streamTsRange);
void sclFreeParam(SScalarParam* param);
int32_t scalarAssignPlaceHolderRes(SColumnInfoData* pResColData, int64_t offset, int64_t rows, int16_t funcId, const void* pExtraParams);
int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst, const void *pExtraParam,
void *streamTsRange);
int32_t scalarCalculateInRange(SNode *pNode, SArray *pBlockList, SScalarParam *pDst, int32_t rowStartIdx,
int32_t rowEndIdx, const void *pExtraParam, void *streamTsRange);
void sclFreeParam(SScalarParam *param);
int32_t scalarAssignPlaceHolderRes(SColumnInfoData *pResColData, int64_t offset, int64_t rows, int16_t funcId,
const void *pExtraParams);
int32_t scalarGetOperatorParamNum(EOperatorType type);
int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type, STypeMod typeMod, int8_t processType);

int32_t vectorGetConvertType(int32_t type1, int32_t type2);
int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, int32_t *overflow, int32_t startIndex, int32_t numOfRows);
int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_t type, STypeMod typeMod, int32_t startIndex, int32_t numOfRows);
int32_t vectorGetConvertType(int32_t type1, int32_t type2);
int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, int32_t *overflow, int32_t startIndex,
int32_t numOfRows);
int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_t type, STypeMod typeMod,
int32_t startIndex, int32_t numOfRows);
STypeMod getConvertTypeMod(int32_t type, const SColumnInfo *pCol1, const SColumnInfo *pCol2);
uint32_t base64BufSize(size_t inputLenBytes);

/* Math functions */
int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
Expand Down Expand Up @@ -90,19 +94,23 @@ int32_t upperFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut
int32_t ltrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t rtrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t md5Function(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOutput);
int32_t md5Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t shaFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t sha2Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t maskFullFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t charFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t asciiFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t positionFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t trimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t replaceFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t repeatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t substrIdxFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t base64Function(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOutput);
int32_t crc32Function(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOutput);
int32_t findInSetFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOutput);
int32_t likeInSetFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOutput);
int32_t regexpInSetFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOutput);
int32_t base64Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t base64FunctionFrom(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t crc32Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t findInSetFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t likeInSetFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t regexpInSetFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

/* Conversion functions */
int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
Expand Down Expand Up @@ -170,7 +178,8 @@ int32_t modeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam
// stream pseudo functions
int32_t streamPseudoScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

int32_t streamCalcCurrWinTimeRange(STimeRangeNode* node, void* pStRtFuncInfo, STimeWindow* pWinRange, bool* winRangeValid, int32_t type);
int32_t streamCalcCurrWinTimeRange(STimeRangeNode *node, void *pStRtFuncInfo, STimeWindow *pWinRange,
bool *winRangeValid, int32_t type);
int32_t scalarCalculateExtWinsTimeRange(STimeRangeNode *pNode, const void *pExtraParam, SExtWinTimeWindow *pWins);

#ifdef __cplusplus
Expand Down
8 changes: 7 additions & 1 deletion include/util/tbase64.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
#define _TD_UTIL_BASE64_H_

#include "os.h"
#include "types.h"

#ifdef __cplusplus
extern "C" {
Expand All @@ -25,8 +26,13 @@ extern "C" {
int32_t base64_decode(const char *value, int32_t inlen, int32_t *outlen, uint8_t **result);
int32_t base64_encode(const uint8_t *value, int32_t vlen, char **result);

void tbase64_encode(uint8_t *out, const uint8_t *input, size_t in_len, VarDataLenT out_len);
int32_t tbase64_decode(uint8_t *out, const uint8_t *input, size_t in_len, VarDataLenT out_len);
uint32_t tbase64_encode_len(size_t in_len);
uint32_t tbase64_decode_len(size_t in_len);

#ifdef __cplusplus
}
#endif

#endif /*_TD_UTIL_BASE64_H_*/
#endif /*_TD_UTIL_BASE64_H_*/
83 changes: 83 additions & 0 deletions include/util/tsha.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
#ifndef _TD_UTIL_SHA_H
#define _TD_UTIL_SHA_H

#include "os.h"

#define SHA1_OUTPUT_LEN 40
#define SHA2_OUTPUT_LEN 128

#ifdef __cplusplus
extern "C" {
#endif

typedef struct {
uint32_t state[5];
uint32_t count[2];
unsigned char buffer[64];
} T_SHA1_CTX;

void tSHA1Transform(uint32_t state[5], const unsigned char buffer[64]);
void tSHA1Init(T_SHA1_CTX *context);
void tSHA1Update(T_SHA1_CTX *context, const unsigned char *data, uint32_t len);
void tSHA1Final(unsigned char digest[20], T_SHA1_CTX *context);
void tSHA1(char *hash_out, const char *str, uint32_t len);

#define SHA224_DIGEST_SIZE (224 / 8)
#define SHA256_DIGEST_SIZE (256 / 8)
#define SHA384_DIGEST_SIZE (384 / 8)
#define SHA512_DIGEST_SIZE (512 / 8)

#define SHA256_BLOCK_SIZE (512 / 8)
#define SHA512_BLOCK_SIZE (1024 / 8)
#define SHA384_BLOCK_SIZE SHA512_BLOCK_SIZE
#define SHA224_BLOCK_SIZE SHA256_BLOCK_SIZE

#ifndef SHA2_TYPES
#define SHA2_TYPES
typedef unsigned char uint8;
typedef unsigned int uint32;
typedef unsigned long long uint64;
Comment on lines +37 to +39
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The custom integer type definitions uint8, uint32, and uint64 are inconsistent with the rest of the codebase, which appears to use standard stdint.h types like uint32_t (as seen in T_SHA1_CTX). Using unsigned int for uint32 is not portable as it's not guaranteed to be 32 bits on all platforms. Since os.h is already included and it provides <stdint.h>, it's better to use the standard fixed-width integer types (uint8_t, uint32_t, uint64_t) for portability and consistency.

Suggested change
typedef unsigned char uint8;
typedef unsigned int uint32;
typedef unsigned long long uint64;
typedef uint8_t uint8;
typedef uint32_t uint32;
typedef uint64_t uint64;

#endif

typedef struct {
uint64 tot_len;
uint64 len;
uint8 block[2 * SHA256_BLOCK_SIZE];
uint32 h[8];
} sha256_ctx;

typedef struct {
uint64 tot_len;
uint64 len;
uint8 block[2 * SHA512_BLOCK_SIZE];
uint64 h[8];
} sha512_ctx;

typedef sha512_ctx sha384_ctx;
typedef sha256_ctx sha224_ctx;

void sha224_init(sha224_ctx *ctx);
void sha224_update(sha224_ctx *ctx, const uint8 *message, uint64 len);
void sha224_final(sha224_ctx *ctx, uint8 *digest);
void sha224(const uint8 *message, uint64 len, uint8 *digest);

void sha256_init(sha256_ctx *ctx);
void sha256_update(sha256_ctx *ctx, const uint8 *message, uint64 len);
void sha256_final(sha256_ctx *ctx, uint8 *digest);
void sha256(const uint8 *message, uint64 len, uint8 *digest);

void sha384_init(sha384_ctx *ctx);
void sha384_update(sha384_ctx *ctx, const uint8 *message, uint64 len);
void sha384_final(sha384_ctx *ctx, uint8 *digest);
void sha384(const uint8 *message, uint64 len, uint8 *digest);

void sha512_init(sha512_ctx *ctx);
void sha512_update(sha512_ctx *ctx, const uint8 *message, uint64 len);
void sha512_final(sha512_ctx *ctx, uint8 *digest);
void sha512(const uint8 *message, uint64 len, uint8 *digest);

#ifdef __cplusplus
}
#endif

#endif /*_TD_UTIL_SHA_H*/
72 changes: 69 additions & 3 deletions include/util/tutil.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#include "tdef.h"
#include "thash.h"
#include "tmd5.h"
#include "tsha.h"

#ifdef __cplusplus
extern "C" {
Expand Down Expand Up @@ -215,17 +216,82 @@ static FORCE_INLINE int32_t taosHashBinary(char *pBuf, int32_t len) {

static FORCE_INLINE int32_t taosCreateMD5Hash(char *pBuf, int32_t len) {
T_MD5_CTX ctx;

tMD5Init(&ctx);
tMD5Update(&ctx, (uint8_t *)pBuf, len);
tMD5Final(&ctx);
char *p = pBuf;
int32_t resLen = 0;

return sprintf(pBuf, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", ctx.digest[0], ctx.digest[1],
ctx.digest[2], ctx.digest[3], ctx.digest[4], ctx.digest[5], ctx.digest[6], ctx.digest[7],
ctx.digest[8], ctx.digest[9], ctx.digest[10], ctx.digest[11], ctx.digest[12], ctx.digest[13],
ctx.digest[14], ctx.digest[15]);
}

static FORCE_INLINE int32_t taosCreateSHA1Hash(char *pBuf, int32_t len) {
uint8_t result[21] = {0};

tSHA1((char *)result, pBuf, len);

return sprintf(pBuf, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", result[0],
result[1], result[2], result[3], result[4], result[5], result[6], result[7], result[8], result[9],
result[10], result[11], result[12], result[13], result[14], result[15], result[16], result[17],
result[18], result[19]);
}

static FORCE_INLINE int32_t taosCreateSHA2Hash(char *pBuf, int32_t len, uint32_t digestSize) {
uint8_t result[2 * SHA512_DIGEST_SIZE + 1] = {0};

switch (digestSize / 8) {
case SHA224_DIGEST_SIZE:
sha224((const uint8_t *)pBuf, len, result);
return sprintf(pBuf,
"%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%"
"02x%02x%02x%02x",
result[0], result[1], result[2], result[3], result[4], result[5], result[6], result[7], result[8],
result[9], result[10], result[11], result[12], result[13], result[14], result[15], result[16],
result[17], result[18], result[19], result[20], result[21], result[22], result[23], result[24],
result[25], result[26], result[27]);
case SHA256_DIGEST_SIZE:
sha256((const uint8_t *)pBuf, len, result);
return sprintf(pBuf,
"%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%"
"02x%02x%02x%02x%02x%02x%02x%02x",
result[0], result[1], result[2], result[3], result[4], result[5], result[6], result[7], result[8],
result[9], result[10], result[11], result[12], result[13], result[14], result[15], result[16],
result[17], result[18], result[19], result[20], result[21], result[22], result[23], result[24],
result[25], result[26], result[27], result[28], result[29], result[30], result[31]);
case SHA384_DIGEST_SIZE:
sha384((const uint8_t *)pBuf, len, result);
return sprintf(pBuf,
"%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%"
"02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x",
result[0], result[1], result[2], result[3], result[4], result[5], result[6], result[7], result[8],
result[9], result[10], result[11], result[12], result[13], result[14], result[15], result[16],
result[17], result[18], result[19], result[20], result[21], result[22], result[23], result[24],
result[25], result[26], result[27], result[28], result[29], result[30], result[31], result[32],
result[33], result[34], result[35], result[36], result[37], result[38], result[39], result[40],
result[41], result[42], result[43], result[44], result[45], result[46], result[47]);
case SHA512_DIGEST_SIZE:
sha512((const uint8_t *)pBuf, len, result);
return sprintf(pBuf,
"%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%"
"02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%"
"02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x",
result[0], result[1], result[2], result[3], result[4], result[5], result[6], result[7], result[8],
result[9], result[10], result[11], result[12], result[13], result[14], result[15], result[16],
result[17], result[18], result[19], result[20], result[21], result[22], result[23], result[24],
result[25], result[26], result[27], result[28], result[29], result[30], result[31], result[32],
result[33], result[34], result[35], result[36], result[37], result[38], result[39], result[40],
result[41], result[42], result[43], result[44], result[45], result[46], result[47], result[48],
result[49], result[50], result[51], result[52], result[53], result[54], result[55], result[56],
result[57], result[58], result[59], result[60], result[61], result[62], result[63]);
default:
break;
}

return 0;
}
Comment on lines +241 to +293
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The functions taosCreateMD5Hash, taosCreateSHA1Hash, and taosCreateSHA2Hash contain very long and hard-to-read sprintf calls to convert byte arrays to hexadecimal strings. This approach is error-prone and involves significant code duplication. I recommend creating a helper function to handle the hex conversion, which would make this code much cleaner, more maintainable, and less susceptible to errors. A similar refactoring should be applied to taosCreateMD5Hash and taosCreateSHA1Hash as well.


static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, int32_t method, int32_t prefix,
int32_t suffix) {
if ((prefix == 0 && suffix == 0) || (tblen <= (prefix + suffix)) || (tblen <= -1 * (prefix + suffix)) ||
Expand Down Expand Up @@ -301,7 +367,7 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen,

#define VND_CHECK_CODE(CODE, LINO, LABEL) TSDB_CHECK_CODE(CODE, LINO, LABEL)

#define TCONTAINER_OF(ptr, type, member) ((type *)((char *)(ptr) - offsetof(type, member)))
#define TCONTAINER_OF(ptr, type, member) ((type *)((char *)(ptr)-offsetof(type, member)))

#define TAOS_GET_TERRNO(code) (terrno == 0 ? code : terrno)

Expand Down
Loading
Loading