diff --git a/README.md b/README.md index 8ad991b09..510361d83 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,11 @@ Beyond statement shapes, the grammar handles nested sub-selects, bind parameters array-literal ambiguity. The complete reference is on the [syntax page](https://jsqlparser.github.io/JSqlParser/syntax.html). +PostgreSQL dollar-quoted strings, including `$tag$…$tag$`, retain their delimiter and +literal body in `StringValue`. Tagged quotes are disabled by default to preserve +identifier parsing. Enable them with `parser.withDialect(Dialect.POSTGRESQL)` or +`parser.withDollarQuotedStringTags(true)`. Untagged `$$…$$` literals remain enabled. + ## Statement classification Any parsed statement can say what it actually does — no second parse, no visitor to write: diff --git a/src/main/java/net/sf/jsqlparser/expression/StringValue.java b/src/main/java/net/sf/jsqlparser/expression/StringValue.java index f02bff523..46bf58810 100644 --- a/src/main/java/net/sf/jsqlparser/expression/StringValue.java +++ b/src/main/java/net/sf/jsqlparser/expression/StringValue.java @@ -42,10 +42,14 @@ public StringValue(String escapedValue) { value = escapedValue.substring(1, escapedValue.length() - 1); quoteStr = "\""; return; - } else if (escapedValue.length() >= 4 && escapedValue.startsWith("$$") - && escapedValue.endsWith("$$")) { - value = escapedValue.substring(2, escapedValue.length() - 2); - quoteStr = "$$"; + } + + String delimiter = getDollarQuoteDelimiter(escapedValue); + if (delimiter != null && escapedValue.length() >= 2 * delimiter.length() + && escapedValue.endsWith(delimiter)) { + quoteStr = delimiter; + value = escapedValue.substring(delimiter.length(), + escapedValue.length() - delimiter.length()); return; } @@ -64,6 +68,32 @@ public StringValue(String escapedValue) { value = escapedValue; } + /** + * Returns the opening PostgreSQL dollar-quote delimiter, or null if there is none. A tag + * follows unquoted identifier rules, excluding dollar signs. This method does not require the + * closing delimiter or inspect the body. + */ + public static String getDollarQuoteDelimiter(String text) { + if (text == null || text.length() < 2 || text.charAt(0) != '$') { + return null; + } + int end = text.indexOf('$', 1); + if (end < 0) { + return null; + } + for (int i = 1; i < end;) { + int character = text.codePointAt(i); + boolean valid = + i == 1 ? Character.isUnicodeIdentifierStart(character) || character == '_' + : Character.isUnicodeIdentifierPart(character); + if (!valid) { + return null; + } + i += Character.charCount(character); + } + return text.substring(0, end + 1); + } + public String getValue() { return value; } @@ -90,6 +120,9 @@ public StringValue setQuoteStr(String quoteStr) { } public String getNotExcapedValue() { + if (quoteStr != null && quoteStr.startsWith("$")) { + return value; + } StringBuilder buffer = new StringBuilder(value); int index = 0; int deletesNum = 0; diff --git a/src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java b/src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java index cd4bfba7e..b5e210d65 100644 --- a/src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java +++ b/src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java @@ -37,7 +37,8 @@ public enum Dialect { Feature.allowHashLineComments, Feature.allowDoubleQuotedStrings), SQLSERVER(AdjacentStringLiterals.OFF, Feature.allowSquareBracketQuotation), POSTGRESQL( - AdjacentStringLiterals.NEWLINE), H2, EXASOL, BIGQUERY( + AdjacentStringLiterals.NEWLINE, + Feature.allowDollarQuotedStringTags), H2, EXASOL, BIGQUERY( AdjacentStringLiterals.WHITESPACE, Feature.allowDoubleQuotedStrings, Feature.allowHashLineComments, @@ -143,6 +144,14 @@ public P withBackslashEscapeCharacter(boolean allowBackslashEscapeCharacter) { return withFeature(Feature.allowBackslashEscapeCharacter, allowBackslashEscapeCharacter); } + /** + * Controls tagged dollar quotes; disabled by default, enabled by the PostgreSQL dialect preset. + * False preserves dollar-containing identifier spellings. + */ + public P withDollarQuotedStringTags(boolean allowDollarQuotedStringTags) { + return withFeature(Feature.allowDollarQuotedStringTags, allowDollarQuotedStringTags); + } + public P withDoubleQuotedStrings() { return withFeature(Feature.allowDoubleQuotedStrings, true); } diff --git a/src/main/java/net/sf/jsqlparser/parser/feature/Feature.java b/src/main/java/net/sf/jsqlparser/parser/feature/Feature.java index d98a0ba1c..dfe607e17 100644 --- a/src/main/java/net/sf/jsqlparser/parser/feature/Feature.java +++ b/src/main/java/net/sf/jsqlparser/parser/feature/Feature.java @@ -808,6 +808,12 @@ public enum Feature { */ allowDoubleQuotedStrings(false), + /** + * Recognizes PostgreSQL $tag$...$tag$ literals; disabled by default to preserve unquoted + * identifiers, enabled by the PostgreSQL dialect preset. Untagged $$ literals are unaffected. + */ + allowDollarQuotedStringTags(false), + /** * concatenates adjacent String Literals: NEWLINE when separated by whitespace with at least one * newline (the SQL standard and PostgreSQL), WHITESPACE across any whitespace (GoogleSQL, diff --git a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt index 860ab3821..8c734e543 100644 --- a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt +++ b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt @@ -1664,33 +1664,27 @@ TOKEN_MGR_DECLS : { return -1; } - private static boolean endsWithDelimiter(Deque windowQueue, String delimiter) { - if (windowQueue.size() != delimiter.length()) { - return false; - } - - int i = 0; - for (char ch : windowQueue) { - if (ch != delimiter.charAt(i++)) { - return false; + /** Scans a literal in linear time without tokenizing or rebuilding its whitespace. */ + public void consumeDollarQuotedString(String closingQuote) { + int[] prefix = new int[closingQuote.length()]; + for (int i = 1, matched = 0; i < closingQuote.length(); i++) { + while (matched > 0 && closingQuote.charAt(i) != closingQuote.charAt(matched)) { + matched = prefix[matched - 1]; } + if (closingQuote.charAt(i) == closingQuote.charAt(matched)) { + matched++; + } + prefix[i] = matched; } - return true; - } - - public void consumeDollarQuotedString(String closingQuote) { - Deque windowQueue = new ArrayDeque(); - int delimiterLength = closingQuote.length(); - try { - while (true) { + int matched = 0; + while (matched < closingQuote.length()) { char ch = input_stream.readChar(); - windowQueue.addLast(ch); - if (windowQueue.size() > delimiterLength) { - windowQueue.removeFirst(); + while (matched > 0 && ch != closingQuote.charAt(matched)) { + matched = prefix[matched - 1]; } - if (endsWithDelimiter(windowQueue, closingQuote)) { - return; + if (ch == closingQuote.charAt(matched)) { + matched++; } } } catch (java.io.IOException e) { @@ -1698,6 +1692,16 @@ TOKEN_MGR_DECLS : { } } + /** Rewinds any identifier suffix consumed by longest-match lexing before scanning the body. */ + private void consumeDollarQuotedToken(Token token, String delimiter) { + input_stream.backup(token.image.length() - delimiter.length()); + consumeDollarQuotedString(delimiter); + token.image = input_stream.GetImage(); + token.kind = charLiteralIndex; + token.endLine = input_stream.getEndLine(); + token.endColumn = input_stream.getEndColumn(); + } + /** * Consumes the body of a block comment after the opening delimiter has been matched, * honouring nesting, up to and including the outermost closing delimiter. Then backs @@ -2407,9 +2411,7 @@ TOKEN: | { - consumeDollarQuotedString(matchedToken.image); - matchedToken.image = input_stream.GetImage(); - matchedToken.kind = charLiteralIndex; + consumeDollarQuotedToken(matchedToken, matchedToken.image); } | // Bare `#` as a binary operator (PostgreSQL bitwise XOR / geometric @@ -2420,6 +2422,13 @@ TOKEN: | ()*) | "$" | ("$" ()*)> { + if (matchedToken.image.charAt(0) == '$' + && Boolean.TRUE.equals(configuration.getValue(Feature.allowDollarQuotedStringTags))) { + String delimiter = StringValue.getDollarQuoteDelimiter(matchedToken.image); + if (delimiter != null) { + consumeDollarQuotedToken(matchedToken, delimiter); + } + } // MySQL `#` line comments (#2499): under the flag an unquoted identifier // ends at its first `#`, the rest of the line becomes a comment via the // stream-level substitution (real MySQL reads `42#24` as `42` plus @@ -2428,7 +2437,7 @@ TOKEN: // that never opted in (the stream is only wired through the feature // consumers / withConfiguration); getValue avoids the String-based // getAsBoolean roundtrip - if (input_stream.featureConfiguration != null + if (matchedToken.kind == S_IDENTIFIER && input_stream.featureConfiguration != null && Boolean.TRUE.equals(configuration.getValue(Feature.allowHashLineComments))) { int hashIndex = matchedToken.image.indexOf('#'); if (hashIndex > 0) { @@ -16680,7 +16689,7 @@ List captureFunctionBody() { tokens.add(tok.image); } foundEnd |= (tok.kind == K_END) - || ( tok.image.trim().startsWith("$$") && tok.image.trim().endsWith("$$")) ; + || (tok.kind == S_CHAR_LITERAL && StringValue.getDollarQuoteDelimiter(tok.image) != null); tok = getNextToken(); } diff --git a/src/test/java/net/sf/jsqlparser/expression/TaggedDollarStringTest.java b/src/test/java/net/sf/jsqlparser/expression/TaggedDollarStringTest.java new file mode 100644 index 000000000..4dd3fa939 --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/expression/TaggedDollarStringTest.java @@ -0,0 +1,190 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2019 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.expression; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; +import java.io.StringReader; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.function.Consumer; +import java.util.stream.Stream; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.parser.AbstractJSqlParser.Dialect; +import net.sf.jsqlparser.parser.CCJSqlParser; +import net.sf.jsqlparser.parser.CCJSqlParserConstants; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.parser.StreamProvider; +import net.sf.jsqlparser.parser.Token; +import net.sf.jsqlparser.schema.Column; +import net.sf.jsqlparser.statement.Statements; +import net.sf.jsqlparser.statement.select.PlainSelect; +import net.sf.jsqlparser.test.TestUtils; +import net.sf.jsqlparser.util.deparser.StatementDeParser; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +class TaggedDollarStringTest { + static Stream tags() { + return Stream.of("", "tag", "Tag_123", "_", "한글", "étiquette"); + } + + @ParameterizedTest + @MethodSource("tags") + void preservesLiteralBodiesAndDelimiters(String tag) throws Exception { + String delimiter = "$" + tag + "$"; + for (String body : List.of("", "abc", "a\nb\r\nc\t ", "x 'one' ''two'' \\ end", + "/* comment */ -- more\n#hash", "[ {\"some\":\"json\",\"with\":\"properties$\"} ]", + "$1 $other$ こんにちは")) { + String literal = delimiter + body + delimiter; + String sql = "SELECT " + literal + " AS value, 2 FROM t"; + PlainSelect select = (PlainSelect) TestUtils.assertSqlCanBeParsedAndDeparsed(sql, true, + parser -> parser.withDialect(Dialect.POSTGRESQL)); + StringValue value = + assertInstanceOf(StringValue.class, select.getSelectItem(0).getExpression()); + assertEquals(body, value.getValue()); + assertEquals(body, value.getNotExcapedValue()); + assertEquals(delimiter, value.getQuoteStr()); + assertEquals(literal, value.toString()); + StringBuilder builder = new StringBuilder(); + select.accept(new StatementDeParser(builder), null); + PlainSelect again = (PlainSelect) CCJSqlParserUtil.parse(builder.toString(), + parser -> parser.withDialect(Dialect.POSTGRESQL)); + assertEquals(body, again.getSelectItem(0).getExpression(StringValue.class).getValue()); + assertEquals(select.toString(), builder.toString()); + } + } + + @Test + void keepsDifferentTagsAndDollarSignsInsideBody() throws Exception { + String body = "$other$ text $Tag$ $$ $1 $t"; + PlainSelect select = + (PlainSelect) CCJSqlParserUtil.parse("SELECT $tag$" + body + "$tag$::text, $1", + parser -> parser.withDialect(Dialect.POSTGRESQL)); + CastExpression cast = select.getSelectItem(0).getExpression(CastExpression.class); + assertEquals(body, ((StringValue) cast.getLeftExpression()).getValue()); + assertInstanceOf(JdbcParameter.class, select.getSelectItem(1).getExpression()); + } + + @Test + void retainsIdentifiersWithPostgreSqlDialect() throws Exception { + PlainSelect select = (PlainSelect) CCJSqlParserUtil + .parse("SELECT $parameter, foo$bar, \"$tag$abc$tag$\", $1 FROM t", + parser -> parser.withDialect(Dialect.POSTGRESQL)); + for (int i = 0; i < 3; i++) { + assertInstanceOf(Column.class, select.getSelectItem(i).getExpression()); + } + assertInstanceOf(JdbcParameter.class, select.getSelectItem(3).getExpression()); + } + + static Stream> identifierConfigurations() { + return Stream.concat( + Stream.>of(parser -> { + }, + parser -> parser.withDialect(Dialect.POSTGRESQL) + .withDollarQuotedStringTags(false)), + Stream.of(Dialect.values()).filter(dialect -> dialect != Dialect.POSTGRESQL) + .map(dialect -> parser -> parser.withDialect(dialect))); + } + + @ParameterizedTest + @MethodSource("identifierConfigurations") + void retainsIdentifiersAndUntaggedLiterals(Consumer configuration) + throws Exception { + for (String identifier : List.of("$tag$abc$tag$", "$tag$identifier")) { + PlainSelect select = + (PlainSelect) CCJSqlParserUtil.parse("SELECT " + identifier, configuration); + assertEquals(identifier, + select.getSelectItem(0).getExpression(Column.class).getColumnName()); + } + PlainSelect untagged = (PlainSelect) CCJSqlParserUtil.parse("SELECT $$text$$", + configuration); + assertEquals("text", untagged.getSelectItem(0).getExpression(StringValue.class).getValue()); + } + + @Test + void supportsExplicitOptInWithoutDialect() throws Exception { + PlainSelect select = (PlainSelect) CCJSqlParserUtil.parse("SELECT $tag$abc$tag$", + parser -> parser.withDollarQuotedStringTags(true)); + assertEquals("abc", select.getSelectItem(0).getExpression(StringValue.class).getValue()); + } + + @Test + void retainsBodyWithOtherLexerOptions() throws Exception { + PlainSelect select = (PlainSelect) CCJSqlParserUtil.parse( + "SELECT $t$#hash\n\\text't$tag$ \"q\"$t$", + parser -> parser.withDialect(Dialect.MYSQL).withDollarQuotedStringTags(true)); + assertEquals("#hash\n\\text't$tag$ \"q\"", + select.getSelectItem(0).getExpression(StringValue.class).getValue()); + } + + @Test + void keepsLineColumnAndAbsoluteTokenPositions() { + String literal = "$tag$a\nb$tag$"; + CCJSqlParser parser = CCJSqlParserUtil.newParser("SELECT " + literal + ", 2") + .withDialect(Dialect.POSTGRESQL); + parser.getNextToken(); + Token value = parser.getNextToken(); + Token comma = parser.getNextToken(); + assertEquals(CCJSqlParserConstants.S_CHAR_LITERAL, value.kind); + assertEquals(literal, value.image); + assertEquals(1, value.beginLine); + assertEquals(8, value.beginColumn); + assertEquals(2, value.endLine); + assertEquals(6, value.endColumn); + assertEquals(8, value.absoluteBegin); + assertEquals(8 + literal.length(), value.absoluteEnd); + assertEquals(value.absoluteEnd, comma.absoluteBegin); + assertEquals(7, comma.beginColumn); + } + + @Test + void recognizesFunctionBodyAndFollowingStatement() throws Exception { + String body = "SELECT 'a;''b'::text;\n"; + String sql = + "CREATE FUNCTION f() RETURNS text AS $fn$" + body + "$fn$ LANGUAGE SQL; SELECT 42;"; + Statements statements = CCJSqlParserUtil.parseStatements(sql, + parser -> parser.withDialect(Dialect.POSTGRESQL)); + assertEquals(2, statements.size()); + assertEquals("SELECT 42", statements.get(1).toString()); + org.junit.jupiter.api.Assertions + .assertTrue(statements.get(0).toString().contains("$fn$" + body + "$fn$")); + assertEquals(2, CCJSqlParserUtil.parseStatements(statements.toString(), + parser -> parser.withDialect(Dialect.POSTGRESQL)).size()); + } + + @Test + @Timeout(10) + void handlesLongBodiesAndOverlappingDelimiterPrefixes() throws Exception { + String body = "$ta$tagX $tagtagX\n".repeat(12000); + String sql = "SELECT $tagtag$" + body + "$tagtag$"; + PlainSelect select = + (PlainSelect) new CCJSqlParser(new StreamProvider(new StringReader(sql))) + .withDialect(Dialect.POSTGRESQL).Statement(); + assertEquals(body, select.getSelectItem(0).getExpression(StringValue.class).getValue()); + PlainSelect streamed = (PlainSelect) CCJSqlParserUtil.newParser( + new java.io.ByteArrayInputStream(sql.getBytes(StandardCharsets.UTF_8)), "UTF-8") + .withDialect(Dialect.POSTGRESQL).Statement(); + assertEquals(body, streamed.getSelectItem(0).getExpression(StringValue.class).getValue()); + } + + @ParameterizedTest + @ValueSource(strings = {"SELECT $tag$missing", "SELECT $Tag$wrong$tag$", "SELECT $t$ends$t", + "SELECT $a$text$b$", "SELECT $$missing"}) + void rejectsUnterminatedOrMismatchedTags(String sql) { + assertThrows(JSQLParserException.class, + () -> CCJSqlParserUtil.parse(sql, + parser -> parser.withDialect(Dialect.POSTGRESQL).withTimeOut(1000))); + } +} diff --git a/src/test/java/net/sf/jsqlparser/statement/select/PostgresTest.java b/src/test/java/net/sf/jsqlparser/statement/select/PostgresTest.java index 218cb580d..fc97c2f4a 100644 --- a/src/test/java/net/sf/jsqlparser/statement/select/PostgresTest.java +++ b/src/test/java/net/sf/jsqlparser/statement/select/PostgresTest.java @@ -14,6 +14,7 @@ import net.sf.jsqlparser.expression.JsonExpression; import net.sf.jsqlparser.expression.StringValue; import net.sf.jsqlparser.expression.operators.relational.Intersects; +import net.sf.jsqlparser.parser.AbstractJSqlParser.Dialect; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.schema.Column; import net.sf.jsqlparser.schema.Table; @@ -108,11 +109,10 @@ void testNextValueIssue1863() throws JSQLParserException { } @Test - @Disabled - // wip void testDollarQuotedText() throws JSQLParserException { String sqlStr = "SELECT $tag$This\nis\na\nselect\ntest\n$tag$ from dual where a=b"; - PlainSelect st = (PlainSelect) CCJSqlParserUtil.parse(sqlStr); + PlainSelect st = (PlainSelect) CCJSqlParserUtil.parse(sqlStr, + parser -> parser.withDialect(Dialect.POSTGRESQL)); StringValue stringValue = st.getSelectItem(0).getExpression(StringValue.class);