mirror of
https://github.com/postgres/postgres.git
synced 2025-10-23 00:02:54 -04:00
Running COPY within a pipeline can break protocol synchronization in multiple ways. psql is limited in terms of result processing if mixing COPY commands with normal queries while controlling a pipeline with the new meta-commands, as an effect of the following reasons: - In COPY mode, the backend ignores additional Sync messages and will not send a matching ReadyForQuery expected by the frontend. Doing a \syncpipeline just after COPY will leave the frontend waiting for a ReadyForQuery message that won't be sent, leaving psql out-of-sync. - libpq automatically sends a Sync with the Copy message which is not tracked in the command queue, creating an unexpected synchronisation point that psql cannot really know about. While it is possible to track such activity for a \copy, this cannot really be done sanely with plain COPY queries. Backend failures during a COPY would leave the pipeline in an aborted state while the backend would be in a clean state, ready to process commands. At the end, fixing those issues would require modifications in how libpq handles pipeline and COPY. So, rather than implementing workarounds in psql to shortcut the libpq internals (with command queue handling for one), and because meta-commands for pipelines in psql are a new feature with COPY in a pipeline having a limited impact compared to other queries, this commit forbids the use of COPY within a pipeline to avoid possible break of protocol synchronisation within psql. If there is a use-case for COPY support within pipelines in libpq, this could always be added in the future, if necessary. Most of the changes of this commit impacts the tests for psql pipelines, removing the tests related to COPY. Some TAP tests still exist for COPY TO/FROM and \copy to/from, to check that that connections are aborted when this operation is attempted. Reported-by: Nikita Kalinin <n.kalinin@postgrespro.ru> Author: Anthonin Bonnefoy <anthonin.bonnefoy@datadoghq.com> Discussion: https://postgr.es/m/AC468509-06E8-4E2A-A4B1-63046A4AC6AB@postgrespro.ru
534 lines
15 KiB
Perl
534 lines
15 KiB
Perl
|
|
# Copyright (c) 2021-2025, PostgreSQL Global Development Group
|
|
|
|
use strict;
|
|
use warnings FATAL => 'all';
|
|
use locale;
|
|
|
|
use PostgreSQL::Test::Cluster;
|
|
use PostgreSQL::Test::Utils;
|
|
use Test::More;
|
|
|
|
program_help_ok('psql');
|
|
program_version_ok('psql');
|
|
program_options_handling_ok('psql');
|
|
|
|
# Execute a psql command and check its output.
|
|
sub psql_like
|
|
{
|
|
local $Test::Builder::Level = $Test::Builder::Level + 1;
|
|
|
|
my ($node, $sql, $expected_stdout, $test_name) = @_;
|
|
|
|
my ($ret, $stdout, $stderr) = $node->psql('postgres', $sql);
|
|
|
|
is($ret, 0, "$test_name: exit code 0");
|
|
is($stderr, '', "$test_name: no stderr");
|
|
like($stdout, $expected_stdout, "$test_name: matches");
|
|
|
|
return;
|
|
}
|
|
|
|
# Execute a psql command and check that it fails and check the stderr.
|
|
sub psql_fails_like
|
|
{
|
|
local $Test::Builder::Level = $Test::Builder::Level + 1;
|
|
|
|
my ($node, $sql, $expected_stderr, $test_name, $replication) = @_;
|
|
|
|
# Use the context of a WAL sender, if requested by the caller.
|
|
$replication = '' unless defined($replication);
|
|
|
|
my ($ret, $stdout, $stderr) =
|
|
$node->psql('postgres', $sql, replication => $replication);
|
|
|
|
isnt($ret, 0, "$test_name: exit code not 0");
|
|
like($stderr, $expected_stderr, "$test_name: matches");
|
|
|
|
return;
|
|
}
|
|
|
|
# test --help=foo, analogous to program_help_ok()
|
|
foreach my $arg (qw(commands variables))
|
|
{
|
|
my ($stdout, $stderr);
|
|
my $result;
|
|
|
|
$result = IPC::Run::run [ 'psql', "--help=$arg" ],
|
|
'>' => \$stdout,
|
|
'2>' => \$stderr;
|
|
ok($result, "psql --help=$arg exit code 0");
|
|
isnt($stdout, '', "psql --help=$arg goes to stdout");
|
|
is($stderr, '', "psql --help=$arg nothing to stderr");
|
|
}
|
|
|
|
my $node = PostgreSQL::Test::Cluster->new('main');
|
|
$node->init(extra => [ '--locale=C', '--encoding=UTF8' ]);
|
|
$node->append_conf(
|
|
'postgresql.conf', q{
|
|
wal_level = 'logical'
|
|
max_replication_slots = 4
|
|
max_wal_senders = 4
|
|
});
|
|
$node->start;
|
|
|
|
psql_like($node, '\copyright', qr/Copyright/, '\copyright');
|
|
psql_like($node, '\help', qr/ALTER/, '\help without arguments');
|
|
psql_like($node, '\help SELECT', qr/SELECT/, '\help with argument');
|
|
|
|
# Test clean handling of unsupported replication command responses
|
|
psql_fails_like(
|
|
$node,
|
|
'START_REPLICATION 0/0',
|
|
qr/unexpected PQresultStatus: 8$/,
|
|
'handling of unexpected PQresultStatus', 'database');
|
|
|
|
# test \timing
|
|
psql_like(
|
|
$node,
|
|
'\timing on
|
|
SELECT 1',
|
|
qr/^1$
|
|
^Time: \d+[.,]\d\d\d ms/m,
|
|
'\timing with successful query');
|
|
|
|
# test \timing with query that fails
|
|
{
|
|
my ($ret, $stdout, $stderr) =
|
|
$node->psql('postgres', "\\timing on\nSELECT error");
|
|
isnt($ret, 0, '\timing with query error: query failed');
|
|
like(
|
|
$stdout,
|
|
qr/^Time: \d+[.,]\d\d\d ms/m,
|
|
'\timing with query error: timing output appears');
|
|
unlike(
|
|
$stdout,
|
|
qr/^Time: 0[.,]000 ms/m,
|
|
'\timing with query error: timing was updated');
|
|
}
|
|
|
|
# test that ENCODING variable is set and that it is updated when
|
|
# client encoding is changed
|
|
psql_like(
|
|
$node,
|
|
'\echo :ENCODING
|
|
set client_encoding = LATIN1;
|
|
\echo :ENCODING',
|
|
qr/^UTF8$
|
|
^LATIN1$/m,
|
|
'ENCODING variable is set and updated');
|
|
|
|
# test LISTEN/NOTIFY
|
|
psql_like(
|
|
$node,
|
|
'LISTEN foo;
|
|
NOTIFY foo;',
|
|
qr/^Asynchronous notification "foo" received from server process with PID \d+\.$/,
|
|
'notification');
|
|
|
|
psql_like(
|
|
$node,
|
|
"LISTEN foo;
|
|
NOTIFY foo, 'bar';",
|
|
qr/^Asynchronous notification "foo" with payload "bar" received from server process with PID \d+\.$/,
|
|
'notification with payload');
|
|
|
|
# test behavior and output on server crash
|
|
my ($ret, $out, $err) = $node->psql('postgres',
|
|
"SELECT 'before' AS running;\n"
|
|
. "SELECT pg_terminate_backend(pg_backend_pid());\n"
|
|
. "SELECT 'AFTER' AS not_running;\n");
|
|
|
|
is($ret, 2, 'server crash: psql exit code');
|
|
like($out, qr/before/, 'server crash: output before crash');
|
|
ok($out !~ qr/AFTER/, 'server crash: no output after crash');
|
|
is( $err,
|
|
'psql:<stdin>:2: FATAL: terminating connection due to administrator command
|
|
psql:<stdin>:2: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
psql:<stdin>:2: error: connection to server was lost',
|
|
'server crash: error message');
|
|
|
|
# test \errverbose
|
|
#
|
|
# (This is not in the regular regression tests because the output
|
|
# contains the source code location and we don't want to have to
|
|
# update that every time it changes.)
|
|
|
|
psql_like(
|
|
$node,
|
|
'SELECT 1;
|
|
\errverbose',
|
|
qr/^1\nThere is no previous error\.$/,
|
|
'\errverbose with no previous error');
|
|
|
|
# There are three main ways to run a query that might affect
|
|
# \errverbose: The normal way, piecemeal retrieval using FETCH_COUNT,
|
|
# and using \gdesc. Test them all.
|
|
|
|
like(
|
|
( $node->psql(
|
|
'postgres',
|
|
"SELECT error;\n\\errverbose",
|
|
on_error_stop => 0))[2],
|
|
qr/\A^psql:<stdin>:1: ERROR: .*$
|
|
^LINE 1: SELECT error;$
|
|
^ *^.*$
|
|
^psql:<stdin>:2: error: ERROR: [0-9A-Z]{5}: .*$
|
|
^LINE 1: SELECT error;$
|
|
^ *^.*$
|
|
^LOCATION: .*$/m,
|
|
'\errverbose after normal query with error');
|
|
|
|
like(
|
|
( $node->psql(
|
|
'postgres',
|
|
"\\set FETCH_COUNT 1\nSELECT error;\n\\errverbose",
|
|
on_error_stop => 0))[2],
|
|
qr/\A^psql:<stdin>:2: ERROR: .*$
|
|
^LINE 1: SELECT error;$
|
|
^ *^.*$
|
|
^psql:<stdin>:3: error: ERROR: [0-9A-Z]{5}: .*$
|
|
^LINE 1: SELECT error;$
|
|
^ *^.*$
|
|
^LOCATION: .*$/m,
|
|
'\errverbose after FETCH_COUNT query with error');
|
|
|
|
like(
|
|
( $node->psql(
|
|
'postgres',
|
|
"SELECT error\\gdesc\n\\errverbose",
|
|
on_error_stop => 0))[2],
|
|
qr/\A^psql:<stdin>:1: ERROR: .*$
|
|
^LINE 1: SELECT error$
|
|
^ *^.*$
|
|
^psql:<stdin>:2: error: ERROR: [0-9A-Z]{5}: .*$
|
|
^LINE 1: SELECT error$
|
|
^ *^.*$
|
|
^LOCATION: .*$/m,
|
|
'\errverbose after \gdesc with error');
|
|
|
|
# Check behavior when using multiple -c and -f switches.
|
|
# Note that we cannot test backend-side errors as tests are unstable in this
|
|
# case: IPC::Run can complain about a SIGPIPE if psql quits before reading a
|
|
# query result.
|
|
my $tempdir = PostgreSQL::Test::Utils::tempdir;
|
|
$node->safe_psql('postgres', "CREATE TABLE tab_psql_single (a int);");
|
|
|
|
# Tests with ON_ERROR_STOP.
|
|
$node->command_ok(
|
|
[
|
|
'psql',
|
|
'--no-psqlrc',
|
|
'--single-transaction',
|
|
'--set' => 'ON_ERROR_STOP=1',
|
|
'--command' => 'INSERT INTO tab_psql_single VALUES (1)',
|
|
'--command' => 'INSERT INTO tab_psql_single VALUES (2)',
|
|
],
|
|
'ON_ERROR_STOP, --single-transaction and multiple -c switches');
|
|
my $row_count =
|
|
$node->safe_psql('postgres', 'SELECT count(*) FROM tab_psql_single');
|
|
is($row_count, '2',
|
|
'--single-transaction commits transaction, ON_ERROR_STOP and multiple -c switches'
|
|
);
|
|
|
|
$node->command_fails(
|
|
[
|
|
'psql',
|
|
'--no-psqlrc',
|
|
'--single-transaction',
|
|
'--set' => 'ON_ERROR_STOP=1',
|
|
'--command' => 'INSERT INTO tab_psql_single VALUES (3)',
|
|
'--command' => "\\copy tab_psql_single FROM '$tempdir/nonexistent'"
|
|
],
|
|
'ON_ERROR_STOP, --single-transaction and multiple -c switches, error');
|
|
$row_count =
|
|
$node->safe_psql('postgres', 'SELECT count(*) FROM tab_psql_single');
|
|
is($row_count, '2',
|
|
'client-side error rolls back transaction, ON_ERROR_STOP and multiple -c switches'
|
|
);
|
|
|
|
# Tests mixing files and commands.
|
|
my $copy_sql_file = "$tempdir/tab_copy.sql";
|
|
my $insert_sql_file = "$tempdir/tab_insert.sql";
|
|
append_to_file($copy_sql_file,
|
|
"\\copy tab_psql_single FROM '$tempdir/nonexistent';");
|
|
append_to_file($insert_sql_file, 'INSERT INTO tab_psql_single VALUES (4);');
|
|
$node->command_ok(
|
|
[
|
|
'psql',
|
|
'--no-psqlrc',
|
|
'--single-transaction',
|
|
'--set' => 'ON_ERROR_STOP=1',
|
|
'--file' => $insert_sql_file,
|
|
'--file' => $insert_sql_file
|
|
],
|
|
'ON_ERROR_STOP, --single-transaction and multiple -f switches');
|
|
$row_count =
|
|
$node->safe_psql('postgres', 'SELECT count(*) FROM tab_psql_single');
|
|
is($row_count, '4',
|
|
'--single-transaction commits transaction, ON_ERROR_STOP and multiple -f switches'
|
|
);
|
|
|
|
$node->command_fails(
|
|
[
|
|
'psql',
|
|
'--no-psqlrc',
|
|
'--single-transaction',
|
|
'--set' => 'ON_ERROR_STOP=1',
|
|
'--file' => $insert_sql_file,
|
|
'--file' => $copy_sql_file
|
|
],
|
|
'ON_ERROR_STOP, --single-transaction and multiple -f switches, error');
|
|
$row_count =
|
|
$node->safe_psql('postgres', 'SELECT count(*) FROM tab_psql_single');
|
|
is($row_count, '4',
|
|
'client-side error rolls back transaction, ON_ERROR_STOP and multiple -f switches'
|
|
);
|
|
|
|
# Tests without ON_ERROR_STOP.
|
|
# The last switch fails on \copy. The command returns a failure and the
|
|
# transaction commits.
|
|
$node->command_fails(
|
|
[
|
|
'psql',
|
|
'--no-psqlrc',
|
|
'--single-transaction',
|
|
'--file' => $insert_sql_file,
|
|
'--file' => $insert_sql_file,
|
|
'--command' => "\\copy tab_psql_single FROM '$tempdir/nonexistent'"
|
|
],
|
|
'no ON_ERROR_STOP, --single-transaction and multiple -f/-c switches');
|
|
$row_count =
|
|
$node->safe_psql('postgres', 'SELECT count(*) FROM tab_psql_single');
|
|
is($row_count, '6',
|
|
'client-side error commits transaction, no ON_ERROR_STOP and multiple -f/-c switches'
|
|
);
|
|
|
|
# The last switch fails on \copy coming from an input file. The command
|
|
# returns a success and the transaction commits.
|
|
$node->command_ok(
|
|
[
|
|
'psql',
|
|
'--no-psqlrc',
|
|
'--single-transaction',
|
|
'--file' => $insert_sql_file,
|
|
'--file' => $insert_sql_file,
|
|
'--file' => $copy_sql_file
|
|
],
|
|
'no ON_ERROR_STOP, --single-transaction and multiple -f switches');
|
|
$row_count =
|
|
$node->safe_psql('postgres', 'SELECT count(*) FROM tab_psql_single');
|
|
is($row_count, '8',
|
|
'client-side error commits transaction, no ON_ERROR_STOP and multiple -f switches'
|
|
);
|
|
|
|
# The last switch makes the command return a success, and the contents of
|
|
# the transaction commit even if there is a failure in-between.
|
|
$node->command_ok(
|
|
[
|
|
'psql',
|
|
'--no-psqlrc',
|
|
'--single-transaction',
|
|
'--command' => 'INSERT INTO tab_psql_single VALUES (5)',
|
|
'--file' => $copy_sql_file,
|
|
'--command' => 'INSERT INTO tab_psql_single VALUES (6)'
|
|
],
|
|
'no ON_ERROR_STOP, --single-transaction and multiple -c switches');
|
|
$row_count =
|
|
$node->safe_psql('postgres', 'SELECT count(*) FROM tab_psql_single');
|
|
is($row_count, '10',
|
|
'client-side error commits transaction, no ON_ERROR_STOP and multiple -c switches'
|
|
);
|
|
|
|
# Test \copy from with DEFAULT option
|
|
$node->safe_psql(
|
|
'postgres',
|
|
"CREATE TABLE copy_default (
|
|
id integer PRIMARY KEY,
|
|
text_value text NOT NULL DEFAULT 'test',
|
|
ts_value timestamp without time zone NOT NULL DEFAULT '2022-07-05'
|
|
)"
|
|
);
|
|
|
|
my $copy_default_sql_file = "$tempdir/copy_default.csv";
|
|
append_to_file($copy_default_sql_file, "1,value,2022-07-04\n");
|
|
append_to_file($copy_default_sql_file, "2,placeholder,2022-07-03\n");
|
|
append_to_file($copy_default_sql_file, "3,placeholder,placeholder\n");
|
|
|
|
psql_like(
|
|
$node,
|
|
"\\copy copy_default from $copy_default_sql_file with (format 'csv', default 'placeholder');
|
|
SELECT * FROM copy_default",
|
|
qr/1\|value\|2022-07-04 00:00:00
|
|
2|test|2022-07-03 00:00:00
|
|
3|test|2022-07-05 00:00:00/,
|
|
'\copy from with DEFAULT');
|
|
|
|
# Check \watch
|
|
# Note: the interval value is parsed with locale-aware strtod()
|
|
psql_like(
|
|
$node, sprintf('SELECT 1 \watch c=3 i=%g', 0.01),
|
|
qr/1\n1\n1/, '\watch with 3 iterations, interval of 0.01');
|
|
|
|
# Sub-millisecond wait works, equivalent to 0.
|
|
psql_like(
|
|
$node, sprintf('SELECT 1 \watch c=3 i=%g', 0.0001),
|
|
qr/1\n1\n1/, '\watch with 3 iterations, interval of 0.0001');
|
|
|
|
# Test zero interval
|
|
psql_like(
|
|
$node, '\set WATCH_INTERVAL 0
|
|
SELECT 1 \watch c=3',
|
|
qr/1\n1\n1/, '\watch with 3 iterations, interval of 0');
|
|
|
|
# Check \watch minimum row count
|
|
psql_fails_like(
|
|
$node,
|
|
'SELECT 3 \watch m=x',
|
|
qr/incorrect minimum row count/,
|
|
'\watch, invalid minimum row setting');
|
|
|
|
psql_fails_like(
|
|
$node,
|
|
'SELECT 3 \watch m=1 min_rows=2',
|
|
qr/minimum row count specified more than once/,
|
|
'\watch, minimum rows is specified more than once');
|
|
|
|
psql_like(
|
|
$node,
|
|
sprintf(
|
|
q{with x as (
|
|
select now()-backend_start AS howlong
|
|
from pg_stat_activity
|
|
where pid = pg_backend_pid()
|
|
) select 123 from x where howlong < '2 seconds' \watch i=%g m=2}, 0.5),
|
|
qr/^123$/,
|
|
'\watch, 2 minimum rows');
|
|
|
|
# Check \watch errors
|
|
psql_fails_like(
|
|
$node,
|
|
'SELECT 1 \watch -10',
|
|
qr/incorrect interval value "-10"/,
|
|
'\watch, negative interval');
|
|
psql_fails_like(
|
|
$node,
|
|
'SELECT 1 \watch 10ab',
|
|
qr/incorrect interval value "10ab"/,
|
|
'\watch, incorrect interval');
|
|
psql_fails_like(
|
|
$node,
|
|
'SELECT 1 \watch 10e400',
|
|
qr/incorrect interval value "10e400"/,
|
|
'\watch, out-of-range interval');
|
|
psql_fails_like(
|
|
$node,
|
|
'SELECT 1 \watch 1 1',
|
|
qr/interval value is specified more than once/,
|
|
'\watch, interval value is specified more than once');
|
|
psql_fails_like(
|
|
$node,
|
|
'SELECT 1 \watch c=1 c=1',
|
|
qr/iteration count is specified more than once/,
|
|
'\watch, iteration count is specified more than once');
|
|
|
|
# Check WATCH_INTERVAL
|
|
psql_like(
|
|
$node,
|
|
'\echo :WATCH_INTERVAL
|
|
\set WATCH_INTERVAL 10
|
|
\echo :WATCH_INTERVAL
|
|
\unset WATCH_INTERVAL
|
|
\echo :WATCH_INTERVAL',
|
|
qr/^2$
|
|
^10$
|
|
^2$/m,
|
|
'WATCH_INTERVAL variable is set and updated');
|
|
psql_fails_like(
|
|
$node,
|
|
'\set WATCH_INTERVAL 1e500',
|
|
qr/is out of range/,
|
|
'WATCH_INTERVAL variable is out of range');
|
|
|
|
# Test \g output piped into a program.
|
|
# The program is perl -pe '' to simply copy the input to the output.
|
|
my $g_file = "$tempdir/g_file_1.out";
|
|
my $perlbin = $^X;
|
|
$perlbin =~ s!\\!/!g if $PostgreSQL::Test::Utils::windows_os;
|
|
my $pipe_cmd = "$perlbin -pe '' >$g_file";
|
|
|
|
psql_like($node, "SELECT 'one' \\g | $pipe_cmd", qr//, "one command \\g");
|
|
my $c1 = slurp_file($g_file);
|
|
like($c1, qr/one/);
|
|
|
|
psql_like($node, "SELECT 'two' \\; SELECT 'three' \\g | $pipe_cmd",
|
|
qr//, "two commands \\g");
|
|
my $c2 = slurp_file($g_file);
|
|
like($c2, qr/two.*three/s);
|
|
|
|
|
|
psql_like(
|
|
$node,
|
|
"\\set SHOW_ALL_RESULTS 0\nSELECT 'four' \\; SELECT 'five' \\g | $pipe_cmd",
|
|
qr//,
|
|
"two commands \\g with only last result");
|
|
my $c3 = slurp_file($g_file);
|
|
like($c3, qr/five/);
|
|
unlike($c3, qr/four/);
|
|
|
|
psql_like($node, "copy (values ('foo'),('bar')) to stdout \\g | $pipe_cmd",
|
|
qr//, "copy output passed to \\g pipe");
|
|
my $c4 = slurp_file($g_file);
|
|
like($c4, qr/foo.*bar/s);
|
|
|
|
# Test COPY within pipelines. These abort the connection from
|
|
# the frontend so they cannot be tested via SQL.
|
|
$node->safe_psql('postgres', 'CREATE TABLE psql_pipeline()');
|
|
my $log_location = -s $node->logfile;
|
|
psql_fails_like(
|
|
$node,
|
|
qq{\\startpipeline
|
|
COPY psql_pipeline FROM STDIN;
|
|
SELECT 'val1';
|
|
\\syncpipeline
|
|
\\endpipeline},
|
|
qr/COPY in a pipeline is not supported, aborting connection/,
|
|
'COPY FROM in pipeline: fails');
|
|
$node->wait_for_log(
|
|
qr/FATAL: .*terminating connection because protocol synchronization was lost/,
|
|
$log_location);
|
|
|
|
# Remove \syncpipeline here.
|
|
psql_fails_like(
|
|
$node,
|
|
qq{\\startpipeline
|
|
COPY psql_pipeline TO STDOUT;
|
|
SELECT 'val1';
|
|
\\endpipeline},
|
|
qr/COPY in a pipeline is not supported, aborting connection/,
|
|
'COPY TO in pipeline: fails');
|
|
|
|
psql_fails_like(
|
|
$node,
|
|
qq{\\startpipeline
|
|
\\copy psql_pipeline from stdin;
|
|
SELECT 'val1';
|
|
\\syncpipeline
|
|
\\endpipeline},
|
|
qr/COPY in a pipeline is not supported, aborting connection/,
|
|
'\copy from in pipeline: fails');
|
|
|
|
# Sync attempt after a COPY TO/FROM.
|
|
psql_fails_like(
|
|
$node,
|
|
qq{\\startpipeline
|
|
\\copy psql_pipeline to stdout;
|
|
\\syncpipeline
|
|
\\endpipeline},
|
|
qr/COPY in a pipeline is not supported, aborting connection/,
|
|
'\copy to in pipeline: fails');
|
|
|
|
done_testing();
|