SQLiteICU.c (18531B)
1 // Copyright (c) Microsoft Corporation. 2 // Licensed under the MIT License. 3 #include <winsqlite/winsqlite3.h> 4 5 // Define values to force static linkage version of extension. 6 // The code below the ICU header include should be changed as little as possible. 7 #define SQLITE_CORE 8 #define SQLITE_ENABLE_ICU 9 #define SQLITE_PRIVATE 10 #define SQLITE_DIRECTONLY 0x000080000 11 #define SQLITE_INNOCUOUS 0x000200000 12 13 #define STDCALL_FOR_INTEROP __stdcall 14 15 // Adapted from the file icu.c (v 1.7 2007/12/13 21:54:11) to use the built-in Windows SQLite and ICU binaries. 16 17 // This file implements an integration between the ICU library 18 // ("International Components for Unicode", an open-source library 19 // for handling unicode data) and SQLite. The integration uses 20 // ICU to provide the following to SQLite: 21 // 22 // * An implementation of the SQL regexp() function (and hence REGEXP 23 // operator) using the ICU uregex_XX() APIs. 24 // 25 // * Implementations of the SQL scalar upper() and lower() functions 26 // for case mapping. 27 // 28 // * Integration of ICU and SQLite collation sequences. 29 // 30 // * An implementation of the LIKE operator that uses ICU to 31 // provide case-independent matching. 32 33 // Include ICU headers 34 #include <icu.h> 35 36 /* #include <assert.h> */ 37 #include <assert.h> 38 39 #ifndef SQLITE_CORE 40 /* #include "sqlite3ext.h" */ 41 SQLITE_EXTENSION_INIT1 42 #else 43 /* #include "sqlite3.h" */ 44 #endif 45 46 /* 47 ** This function is called when an ICU function called from within 48 ** the implementation of an SQL scalar function returns an error. 49 ** 50 ** The scalar function context passed as the first argument is 51 ** loaded with an error message based on the following two args. 52 */ 53 static void icuFunctionError( 54 sqlite3_context* pCtx, /* SQLite scalar function context */ 55 const char* zName, /* Name of ICU function that failed */ 56 UErrorCode e /* Error code returned by ICU function */ 57 ) { 58 char zBuf[128]; 59 sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e)); 60 zBuf[127] = '\0'; 61 sqlite3_result_error(pCtx, zBuf, -1); 62 } 63 64 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) 65 66 /* 67 ** Maximum length (in bytes) of the pattern in a LIKE or GLOB 68 ** operator. 69 */ 70 #ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH 71 # define SQLITE_MAX_LIKE_PATTERN_LENGTH 50000 72 #endif 73 74 /* 75 ** Version of sqlite3_free() that is always a function, never a macro. 76 */ 77 static void STDCALL_FOR_INTEROP xFree(void* p) { 78 sqlite3_free(p); 79 } 80 81 /* 82 ** This lookup table is used to help decode the first byte of 83 ** a multi-byte UTF8 character. It is copied here from SQLite source 84 ** code file utf8.c. 85 */ 86 static const unsigned char icuUtf8Trans1[] = { 87 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 88 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 89 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 90 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 91 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 92 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 93 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 94 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, 95 }; 96 97 #define SQLITE_ICU_READ_UTF8(zIn, c) \ 98 c = *(zIn++); \ 99 if( c>=0xc0 ){ \ 100 c = icuUtf8Trans1[c-0xc0]; \ 101 while( (*zIn & 0xc0)==0x80 ){ \ 102 c = (c<<6) + (0x3f & *(zIn++)); \ 103 } \ 104 } 105 106 #define SQLITE_ICU_SKIP_UTF8(zIn) \ 107 assert( *zIn ); \ 108 if( *(zIn++)>=0xc0 ){ \ 109 while( (*zIn & 0xc0)==0x80 ){zIn++;} \ 110 } 111 112 113 /* 114 ** Compare two UTF-8 strings for equality where the first string is 115 ** a "LIKE" expression. Return true (1) if they are the same and 116 ** false (0) if they are different. 117 */ 118 static int icuLikeCompare( 119 const uint8_t* zPattern, /* LIKE pattern */ 120 const uint8_t* zString, /* The UTF-8 string to compare against */ 121 const UChar32 uEsc /* The escape character */ 122 ) { 123 static const uint32_t MATCH_ONE = (uint32_t)'_'; 124 static const uint32_t MATCH_ALL = (uint32_t)'%'; 125 126 int prevEscape = 0; /* True if the previous character was uEsc */ 127 128 while (1) { 129 130 /* Read (and consume) the next character from the input pattern. */ 131 uint32_t uPattern; 132 SQLITE_ICU_READ_UTF8(zPattern, uPattern); 133 if (uPattern == 0) break; 134 135 /* There are now 4 possibilities: 136 ** 137 ** 1. uPattern is an unescaped match-all character "%", 138 ** 2. uPattern is an unescaped match-one character "_", 139 ** 3. uPattern is an unescaped escape character, or 140 ** 4. uPattern is to be handled as an ordinary character 141 */ 142 if (!prevEscape && uPattern == MATCH_ALL) { 143 /* Case 1. */ 144 uint8_t c; 145 146 /* Skip any MATCH_ALL or MATCH_ONE characters that follow a 147 ** MATCH_ALL. For each MATCH_ONE, skip one character in the 148 ** test string. 149 */ 150 while ((c = *zPattern) == MATCH_ALL || c == MATCH_ONE) { 151 if (c == MATCH_ONE) { 152 if (*zString == 0) return 0; 153 SQLITE_ICU_SKIP_UTF8(zString); 154 } 155 zPattern++; 156 } 157 158 if (*zPattern == 0) return 1; 159 160 while (*zString) { 161 if (icuLikeCompare(zPattern, zString, uEsc)) { 162 return 1; 163 } 164 SQLITE_ICU_SKIP_UTF8(zString); 165 } 166 return 0; 167 168 } 169 else if (!prevEscape && uPattern == MATCH_ONE) { 170 /* Case 2. */ 171 if (*zString == 0) return 0; 172 SQLITE_ICU_SKIP_UTF8(zString); 173 174 } 175 else if (!prevEscape && uPattern == (uint32_t)uEsc) { 176 /* Case 3. */ 177 prevEscape = 1; 178 179 } 180 else { 181 /* Case 4. */ 182 uint32_t uString; 183 SQLITE_ICU_READ_UTF8(zString, uString); 184 uString = (uint32_t)u_foldCase((UChar32)uString, U_FOLD_CASE_DEFAULT); 185 uPattern = (uint32_t)u_foldCase((UChar32)uPattern, U_FOLD_CASE_DEFAULT); 186 if (uString != uPattern) { 187 return 0; 188 } 189 prevEscape = 0; 190 } 191 } 192 193 return *zString == 0; 194 } 195 196 /* 197 ** Implementation of the like() SQL function. This function implements 198 ** the build-in LIKE operator. The first argument to the function is the 199 ** pattern and the second argument is the string. So, the SQL statements: 200 ** 201 ** A LIKE B 202 ** 203 ** is implemented as like(B, A). If there is an escape character E, 204 ** 205 ** A LIKE B ESCAPE E 206 ** 207 ** is mapped to like(B, A, E). 208 */ 209 static void STDCALL_FOR_INTEROP icuLikeFunc( 210 sqlite3_context* context, 211 int argc, 212 sqlite3_value** argv 213 ) { 214 const unsigned char* zA = sqlite3_value_text(argv[0]); 215 const unsigned char* zB = sqlite3_value_text(argv[1]); 216 UChar32 uEsc = 0; 217 218 /* Limit the length of the LIKE or GLOB pattern to avoid problems 219 ** of deep recursion and N*N behavior in patternCompare(). 220 */ 221 if (sqlite3_value_bytes(argv[0]) > SQLITE_MAX_LIKE_PATTERN_LENGTH) { 222 sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1); 223 return; 224 } 225 226 227 if (argc == 3) { 228 /* The escape character string must consist of a single UTF-8 character. 229 ** Otherwise, return an error. 230 */ 231 int nE = sqlite3_value_bytes(argv[2]); 232 const unsigned char* zE = sqlite3_value_text(argv[2]); 233 int i = 0; 234 if (zE == 0) return; 235 U8_NEXT(zE, i, nE, uEsc); 236 if (i != nE) { 237 sqlite3_result_error(context, 238 "ESCAPE expression must be a single character", -1); 239 return; 240 } 241 } 242 243 if (zA && zB) { 244 sqlite3_result_int(context, icuLikeCompare(zA, zB, uEsc)); 245 } 246 } 247 248 /* 249 ** Function to delete compiled regexp objects. Registered as 250 ** a destructor function with sqlite3_set_auxdata(). 251 */ 252 static void STDCALL_FOR_INTEROP icuRegexpDelete(void* p) { 253 URegularExpression* pExpr = (URegularExpression*)p; 254 uregex_close(pExpr); 255 } 256 257 /* 258 ** Implementation of SQLite REGEXP operator. This scalar function takes 259 ** two arguments. The first is a regular expression pattern to compile 260 ** the second is a string to match against that pattern. If either 261 ** argument is an SQL NULL, then NULL Is returned. Otherwise, the result 262 ** is 1 if the string matches the pattern, or 0 otherwise. 263 ** 264 ** SQLite maps the regexp() function to the regexp() operator such 265 ** that the following two are equivalent: 266 ** 267 ** zString REGEXP zPattern 268 ** regexp(zPattern, zString) 269 ** 270 ** Uses the following ICU regexp APIs: 271 ** 272 ** uregex_open() 273 ** uregex_matches() 274 ** uregex_close() 275 */ 276 static void STDCALL_FOR_INTEROP icuRegexpFunc(sqlite3_context* p, int nArg, sqlite3_value** apArg) { 277 UErrorCode status = U_ZERO_ERROR; 278 URegularExpression* pExpr; 279 UBool res; 280 const UChar* zString = sqlite3_value_text16(apArg[1]); 281 282 (void)nArg; /* Unused parameter */ 283 284 /* If the left hand side of the regexp operator is NULL, 285 ** then the result is also NULL. 286 */ 287 if (!zString) { 288 return; 289 } 290 291 pExpr = sqlite3_get_auxdata(p, 0); 292 if (!pExpr) { 293 const UChar* zPattern = sqlite3_value_text16(apArg[0]); 294 if (!zPattern) { 295 return; 296 } 297 pExpr = uregex_open(zPattern, -1, 0, 0, &status); 298 299 if (U_SUCCESS(status)) { 300 sqlite3_set_auxdata(p, 0, pExpr, icuRegexpDelete); 301 } 302 else { 303 assert(!pExpr); 304 icuFunctionError(p, "uregex_open", status); 305 return; 306 } 307 } 308 309 /* Configure the text that the regular expression operates on. */ 310 uregex_setText(pExpr, zString, -1, &status); 311 if (!U_SUCCESS(status)) { 312 icuFunctionError(p, "uregex_setText", status); 313 return; 314 } 315 316 /* Attempt the match */ 317 res = uregex_matches(pExpr, 0, &status); 318 if (!U_SUCCESS(status)) { 319 icuFunctionError(p, "uregex_matches", status); 320 return; 321 } 322 323 /* Set the text that the regular expression operates on to a NULL 324 ** pointer. This is not really necessary, but it is tidier than 325 ** leaving the regular expression object configured with an invalid 326 ** pointer after this function returns. 327 */ 328 uregex_setText(pExpr, 0, 0, &status); 329 330 /* Return 1 or 0. */ 331 sqlite3_result_int(p, res ? 1 : 0); 332 } 333 334 /* 335 ** Implementations of scalar functions for case mapping - upper() and 336 ** lower(). Function upper() converts its input to upper-case (ABC). 337 ** Function lower() converts to lower-case (abc). 338 ** 339 ** ICU provides two types of case mapping, "general" case mapping and 340 ** "language specific". Refer to ICU documentation for the differences 341 ** between the two. 342 ** 343 ** To utilise "general" case mapping, the upper() or lower() scalar 344 ** functions are invoked with one argument: 345 ** 346 ** upper('ABC') -> 'abc' 347 ** lower('abc') -> 'ABC' 348 ** 349 ** To access ICU "language specific" case mapping, upper() or lower() 350 ** should be invoked with two arguments. The second argument is the name 351 ** of the locale to use. Passing an empty string ("") or SQL NULL value 352 ** as the second argument is the same as invoking the 1 argument version 353 ** of upper() or lower(). 354 ** 355 ** lower('I', 'en_us') -> 'i' 356 ** lower('I', 'tr_tr') -> '\u131' (small dotless i) 357 ** 358 ** http://www.icu-project.org/userguide/posix.html#case_mappings 359 */ 360 static void STDCALL_FOR_INTEROP icuCaseFunc16(sqlite3_context* p, int nArg, sqlite3_value** apArg) { 361 const UChar* zInput; /* Pointer to input string */ 362 UChar* zOutput = 0; /* Pointer to output buffer */ 363 int nInput; /* Size of utf-16 input string in bytes */ 364 int nOut; /* Size of output buffer in bytes */ 365 int cnt; 366 int bToUpper; /* True for toupper(), false for tolower() */ 367 UErrorCode status; 368 const char* zLocale = 0; 369 370 assert(nArg == 1 || nArg == 2); 371 bToUpper = (sqlite3_user_data(p) != 0); 372 if (nArg == 2) { 373 zLocale = (const char*)sqlite3_value_text(apArg[1]); 374 } 375 376 zInput = sqlite3_value_text16(apArg[0]); 377 if (!zInput) { 378 return; 379 } 380 nOut = nInput = sqlite3_value_bytes16(apArg[0]); 381 if (nOut == 0) { 382 sqlite3_result_text16(p, "", 0, SQLITE_STATIC); 383 return; 384 } 385 386 for (cnt = 0; cnt < 2; cnt++) { 387 UChar* zNew = sqlite3_realloc(zOutput, nOut); 388 if (zNew == 0) { 389 sqlite3_free(zOutput); 390 sqlite3_result_error_nomem(p); 391 return; 392 } 393 zOutput = zNew; 394 status = U_ZERO_ERROR; 395 if (bToUpper) { 396 nOut = 2 * u_strToUpper(zOutput, nOut / 2, zInput, nInput / 2, zLocale, &status); 397 } 398 else { 399 nOut = 2 * u_strToLower(zOutput, nOut / 2, zInput, nInput / 2, zLocale, &status); 400 } 401 402 if (U_SUCCESS(status)) { 403 sqlite3_result_text16(p, zOutput, nOut, xFree); 404 } 405 else if (status == U_BUFFER_OVERFLOW_ERROR) { 406 assert(cnt == 0); 407 continue; 408 } 409 else { 410 icuFunctionError(p, bToUpper ? "u_strToUpper" : "u_strToLower", status); 411 } 412 return; 413 } 414 assert(0); /* Unreachable */ 415 } 416 417 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */ 418 419 /* 420 ** Collation sequence destructor function. The pCtx argument points to 421 ** a UCollator structure previously allocated using ucol_open(). 422 */ 423 static void STDCALL_FOR_INTEROP icuCollationDel(void* pCtx) { 424 UCollator* p = (UCollator*)pCtx; 425 ucol_close(p); 426 } 427 428 /* 429 ** Collation sequence comparison function. The pCtx argument points to 430 ** a UCollator structure previously allocated using ucol_open(). 431 */ 432 static int STDCALL_FOR_INTEROP icuCollationColl( 433 void* pCtx, 434 int nLeft, 435 const void* zLeft, 436 int nRight, 437 const void* zRight 438 ) { 439 UCollationResult res; 440 UCollator* p = (UCollator*)pCtx; 441 res = ucol_strcoll(p, (UChar*)zLeft, nLeft / 2, (UChar*)zRight, nRight / 2); 442 switch (res) { 443 case UCOL_LESS: return -1; 444 case UCOL_GREATER: return +1; 445 case UCOL_EQUAL: return 0; 446 } 447 assert(!"Unexpected return value from ucol_strcoll()"); 448 return 0; 449 } 450 451 /* 452 ** Implementation of the scalar function icu_load_collation(). 453 ** 454 ** This scalar function is used to add ICU collation based collation 455 ** types to an SQLite database connection. It is intended to be called 456 ** as follows: 457 ** 458 ** SELECT icu_load_collation(<locale>, <collation-name>); 459 ** 460 ** Where <locale> is a string containing an ICU locale identifier (i.e. 461 ** "en_AU", "tr_TR" etc.) and <collation-name> is the name of the 462 ** collation sequence to create. 463 */ 464 static void STDCALL_FOR_INTEROP icuLoadCollation( 465 sqlite3_context* p, 466 int nArg, 467 sqlite3_value** apArg 468 ) { 469 sqlite3* db = (sqlite3*)sqlite3_user_data(p); 470 UErrorCode status = U_ZERO_ERROR; 471 const char* zLocale; /* Locale identifier - (eg. "jp_JP") */ 472 const char* zName; /* SQL Collation sequence name (eg. "japanese") */ 473 UCollator* pUCollator; /* ICU library collation object */ 474 int rc; /* Return code from sqlite3_create_collation_x() */ 475 476 assert(nArg == 2); 477 (void)nArg; /* Unused parameter */ 478 zLocale = (const char*)sqlite3_value_text(apArg[0]); 479 zName = (const char*)sqlite3_value_text(apArg[1]); 480 481 if (!zLocale || !zName) { 482 return; 483 } 484 485 pUCollator = ucol_open(zLocale, &status); 486 if (!U_SUCCESS(status)) { 487 icuFunctionError(p, "ucol_open", status); 488 return; 489 } 490 assert(p); 491 492 rc = sqlite3_create_collation_v2(db, zName, SQLITE_UTF16, (void*)pUCollator, 493 icuCollationColl, icuCollationDel 494 ); 495 if (rc != SQLITE_OK) { 496 ucol_close(pUCollator); 497 sqlite3_result_error(p, "Error registering collation function", -1); 498 } 499 } 500 501 /* 502 ** Register the ICU extension functions with database db. 503 */ 504 SQLITE_PRIVATE int sqlite3IcuInit(sqlite3* db) { 505 # define SQLITEICU_EXTRAFLAGS (SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS) 506 static const struct IcuScalar { 507 const char* zName; /* Function name */ 508 unsigned char nArg; /* Number of arguments */ 509 unsigned int enc; /* Optimal text encoding */ 510 unsigned char iContext; /* sqlite3_user_data() context */ 511 void (STDCALL_FOR_INTEROP *xFunc)(sqlite3_context*, int, sqlite3_value**); 512 } scalars[] = { 513 {"icu_load_collation",2,SQLITE_UTF8 | SQLITE_DIRECTONLY,1, icuLoadCollation}, 514 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) 515 {"regexp", 2, SQLITE_ANY | SQLITEICU_EXTRAFLAGS, 0, icuRegexpFunc}, 516 {"lower", 1, SQLITE_UTF16 | SQLITEICU_EXTRAFLAGS, 0, icuCaseFunc16}, 517 {"lower", 2, SQLITE_UTF16 | SQLITEICU_EXTRAFLAGS, 0, icuCaseFunc16}, 518 {"upper", 1, SQLITE_UTF16 | SQLITEICU_EXTRAFLAGS, 1, icuCaseFunc16}, 519 {"upper", 2, SQLITE_UTF16 | SQLITEICU_EXTRAFLAGS, 1, icuCaseFunc16}, 520 {"lower", 1, SQLITE_UTF8 | SQLITEICU_EXTRAFLAGS, 0, icuCaseFunc16}, 521 {"lower", 2, SQLITE_UTF8 | SQLITEICU_EXTRAFLAGS, 0, icuCaseFunc16}, 522 {"upper", 1, SQLITE_UTF8 | SQLITEICU_EXTRAFLAGS, 1, icuCaseFunc16}, 523 {"upper", 2, SQLITE_UTF8 | SQLITEICU_EXTRAFLAGS, 1, icuCaseFunc16}, 524 {"like", 2, SQLITE_UTF8 | SQLITEICU_EXTRAFLAGS, 0, icuLikeFunc}, 525 {"like", 3, SQLITE_UTF8 | SQLITEICU_EXTRAFLAGS, 0, icuLikeFunc}, 526 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */ 527 }; 528 int rc = SQLITE_OK; 529 int i; 530 531 for (i = 0; rc == SQLITE_OK && i < (int)(sizeof(scalars) / sizeof(scalars[0])); i++) { 532 const struct IcuScalar* p = &scalars[i]; 533 rc = sqlite3_create_function( 534 db, p->zName, p->nArg, p->enc, 535 p->iContext ? (void*)db : (void*)0, 536 p->xFunc, 0, 0 537 ); 538 } 539 540 return rc; 541 } 542 543 #ifndef SQLITE_CORE 544 #ifdef _WIN32 545 __declspec(dllexport) 546 #endif 547 SQLITE_API int sqlite3_icu_init( 548 sqlite3* db, 549 char** pzErrMsg, 550 const sqlite3_api_routines* pApi 551 ) { 552 SQLITE_EXTENSION_INIT2(pApi) 553 return sqlite3IcuInit(db); 554 } 555 #endif 556 557 /************** End of icu.c *************************************************/