mirror of
https://github.com/postgres/postgres.git
synced 2025-05-13 01:13:08 -04:00
a6525588b7
SQL includes provisions for numeric Unicode escapes in string literals and identifiers. Previously we only accepted those if they represented ASCII characters or the server encoding was UTF-8, making the conversion to internal form trivial. This patch adjusts things so that we'll call the appropriate encoding conversion function in less-trivial cases, allowing the escape sequence to be accepted so long as it corresponds to some character available in the server encoding. This also applies to processing of Unicode escapes in JSONB. However, the old restriction still applies to client-side JSON processing, since that hasn't got access to the server's encoding conversion infrastructure. This patch includes some lexer infrastructure that simplifies throwing errors with error cursors pointing into the middle of a string (or other complex token). For the moment I only used it for errors relating to Unicode escapes, but we might later expand the usage to some other cases. Patch by me, reviewed by John Naylor. Discussion: https://postgr.es/m/2393.1578958316@sss.pgh.pa.us
79 lines
3.4 KiB
SQL
79 lines
3.4 KiB
SQL
--
|
|
-- encoding-sensitive tests for json and jsonb
|
|
--
|
|
|
|
-- We provide expected-results files for UTF8 (json_encoding.out)
|
|
-- and for SQL_ASCII (json_encoding_1.out). Skip otherwise.
|
|
SELECT getdatabaseencoding() NOT IN ('UTF8', 'SQL_ASCII')
|
|
AS skip_test \gset
|
|
\if :skip_test
|
|
\quit
|
|
\endif
|
|
|
|
SELECT getdatabaseencoding(); -- just to label the results files
|
|
|
|
-- first json
|
|
|
|
-- basic unicode input
|
|
SELECT '"\u"'::json; -- ERROR, incomplete escape
|
|
SELECT '"\u00"'::json; -- ERROR, incomplete escape
|
|
SELECT '"\u000g"'::json; -- ERROR, g is not a hex digit
|
|
SELECT '"\u0000"'::json; -- OK, legal escape
|
|
SELECT '"\uaBcD"'::json; -- OK, uppercase and lower case both OK
|
|
|
|
-- handling of unicode surrogate pairs
|
|
|
|
select json '{ "a": "\ud83d\ude04\ud83d\udc36" }' -> 'a' as correct_in_utf8;
|
|
select json '{ "a": "\ud83d\ud83d" }' -> 'a'; -- 2 high surrogates in a row
|
|
select json '{ "a": "\ude04\ud83d" }' -> 'a'; -- surrogates in wrong order
|
|
select json '{ "a": "\ud83dX" }' -> 'a'; -- orphan high surrogate
|
|
select json '{ "a": "\ude04X" }' -> 'a'; -- orphan low surrogate
|
|
|
|
--handling of simple unicode escapes
|
|
|
|
select json '{ "a": "the Copyright \u00a9 sign" }' as correct_in_utf8;
|
|
select json '{ "a": "dollar \u0024 character" }' as correct_everywhere;
|
|
select json '{ "a": "dollar \\u0024 character" }' as not_an_escape;
|
|
select json '{ "a": "null \u0000 escape" }' as not_unescaped;
|
|
select json '{ "a": "null \\u0000 escape" }' as not_an_escape;
|
|
|
|
select json '{ "a": "the Copyright \u00a9 sign" }' ->> 'a' as correct_in_utf8;
|
|
select json '{ "a": "dollar \u0024 character" }' ->> 'a' as correct_everywhere;
|
|
select json '{ "a": "dollar \\u0024 character" }' ->> 'a' as not_an_escape;
|
|
select json '{ "a": "null \u0000 escape" }' ->> 'a' as fails;
|
|
select json '{ "a": "null \\u0000 escape" }' ->> 'a' as not_an_escape;
|
|
|
|
-- then jsonb
|
|
|
|
-- basic unicode input
|
|
SELECT '"\u"'::jsonb; -- ERROR, incomplete escape
|
|
SELECT '"\u00"'::jsonb; -- ERROR, incomplete escape
|
|
SELECT '"\u000g"'::jsonb; -- ERROR, g is not a hex digit
|
|
SELECT '"\u0045"'::jsonb; -- OK, legal escape
|
|
SELECT '"\u0000"'::jsonb; -- ERROR, we don't support U+0000
|
|
-- use octet_length here so we don't get an odd unicode char in the
|
|
-- output
|
|
SELECT octet_length('"\uaBcD"'::jsonb::text); -- OK, uppercase and lower case both OK
|
|
|
|
-- handling of unicode surrogate pairs
|
|
|
|
SELECT octet_length((jsonb '{ "a": "\ud83d\ude04\ud83d\udc36" }' -> 'a')::text) AS correct_in_utf8;
|
|
SELECT jsonb '{ "a": "\ud83d\ud83d" }' -> 'a'; -- 2 high surrogates in a row
|
|
SELECT jsonb '{ "a": "\ude04\ud83d" }' -> 'a'; -- surrogates in wrong order
|
|
SELECT jsonb '{ "a": "\ud83dX" }' -> 'a'; -- orphan high surrogate
|
|
SELECT jsonb '{ "a": "\ude04X" }' -> 'a'; -- orphan low surrogate
|
|
|
|
-- handling of simple unicode escapes
|
|
|
|
SELECT jsonb '{ "a": "the Copyright \u00a9 sign" }' as correct_in_utf8;
|
|
SELECT jsonb '{ "a": "dollar \u0024 character" }' as correct_everywhere;
|
|
SELECT jsonb '{ "a": "dollar \\u0024 character" }' as not_an_escape;
|
|
SELECT jsonb '{ "a": "null \u0000 escape" }' as fails;
|
|
SELECT jsonb '{ "a": "null \\u0000 escape" }' as not_an_escape;
|
|
|
|
SELECT jsonb '{ "a": "the Copyright \u00a9 sign" }' ->> 'a' as correct_in_utf8;
|
|
SELECT jsonb '{ "a": "dollar \u0024 character" }' ->> 'a' as correct_everywhere;
|
|
SELECT jsonb '{ "a": "dollar \\u0024 character" }' ->> 'a' as not_an_escape;
|
|
SELECT jsonb '{ "a": "null \u0000 escape" }' ->> 'a' as fails;
|
|
SELECT jsonb '{ "a": "null \\u0000 escape" }' ->> 'a' as not_an_escape;
|