Age Owner Branch data TLA Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * basebackup.c
4 : : * code for taking a base backup and streaming it to a standby
5 : : *
6 : : * Portions Copyright (c) 2010-2025, PostgreSQL Global Development Group
7 : : *
8 : : * IDENTIFICATION
9 : : * src/backend/backup/basebackup.c
10 : : *
11 : : *-------------------------------------------------------------------------
12 : : */
13 : : #include "postgres.h"
14 : :
15 : : #include <sys/stat.h>
16 : : #include <unistd.h>
17 : : #include <time.h>
18 : :
19 : : #include "access/xlog_internal.h"
20 : : #include "access/xlogbackup.h"
21 : : #include "backup/backup_manifest.h"
22 : : #include "backup/basebackup.h"
23 : : #include "backup/basebackup_incremental.h"
24 : : #include "backup/basebackup_sink.h"
25 : : #include "backup/basebackup_target.h"
26 : : #include "catalog/pg_tablespace_d.h"
27 : : #include "commands/defrem.h"
28 : : #include "common/compression.h"
29 : : #include "common/file_perm.h"
30 : : #include "common/file_utils.h"
31 : : #include "lib/stringinfo.h"
32 : : #include "miscadmin.h"
33 : : #include "nodes/pg_list.h"
34 : : #include "pgstat.h"
35 : : #include "pgtar.h"
36 : : #include "postmaster/syslogger.h"
37 : : #include "postmaster/walsummarizer.h"
38 : : #include "replication/slot.h"
39 : : #include "replication/walsender.h"
40 : : #include "replication/walsender_private.h"
41 : : #include "storage/bufpage.h"
42 : : #include "storage/checksum.h"
43 : : #include "storage/dsm_impl.h"
44 : : #include "storage/ipc.h"
45 : : #include "storage/reinit.h"
46 : : #include "utils/builtins.h"
47 : : #include "utils/guc.h"
48 : : #include "utils/ps_status.h"
49 : : #include "utils/relcache.h"
50 : : #include "utils/resowner.h"
51 : :
52 : : /*
53 : : * How much data do we want to send in one CopyData message? Note that
54 : : * this may also result in reading the underlying files in chunks of this
55 : : * size.
56 : : *
57 : : * NB: The buffer size is required to be a multiple of the system block
58 : : * size, so use that value instead if it's bigger than our preference.
59 : : */
60 : : #define SINK_BUFFER_LENGTH Max(32768, BLCKSZ)
61 : :
62 : : typedef struct
63 : : {
64 : : const char *label;
65 : : bool progress;
66 : : bool fastcheckpoint;
67 : : bool nowait;
68 : : bool includewal;
69 : : bool incremental;
70 : : uint32 maxrate;
71 : : bool sendtblspcmapfile;
72 : : bool send_to_client;
73 : : bool use_copytblspc;
74 : : BaseBackupTargetHandle *target_handle;
75 : : backup_manifest_option manifest;
76 : : pg_compress_algorithm compression;
77 : : pg_compress_specification compression_specification;
78 : : pg_checksum_type manifest_checksum_type;
79 : : } basebackup_options;
80 : :
81 : : static int64 sendTablespace(bbsink *sink, char *path, Oid spcoid, bool sizeonly,
82 : : struct backup_manifest_info *manifest,
83 : : IncrementalBackupInfo *ib);
84 : : static int64 sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
85 : : List *tablespaces, bool sendtblspclinks,
86 : : backup_manifest_info *manifest, Oid spcoid,
87 : : IncrementalBackupInfo *ib);
88 : : static bool sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
89 : : struct stat *statbuf, bool missing_ok,
90 : : Oid dboid, Oid spcoid, RelFileNumber relfilenumber,
91 : : unsigned segno,
92 : : backup_manifest_info *manifest,
93 : : unsigned num_incremental_blocks,
94 : : BlockNumber *incremental_blocks,
95 : : unsigned truncation_block_length);
96 : : static off_t read_file_data_into_buffer(bbsink *sink,
97 : : const char *readfilename, int fd,
98 : : off_t offset, size_t length,
99 : : BlockNumber blkno,
100 : : bool verify_checksum,
101 : : int *checksum_failures);
102 : : static void push_to_sink(bbsink *sink, pg_checksum_context *checksum_ctx,
103 : : size_t *bytes_done, void *data, size_t length);
104 : : static bool verify_page_checksum(Page page, XLogRecPtr start_lsn,
105 : : BlockNumber blkno,
106 : : uint16 *expected_checksum);
107 : : static void sendFileWithContent(bbsink *sink, const char *filename,
108 : : const char *content, int len,
109 : : backup_manifest_info *manifest);
110 : : static int64 _tarWriteHeader(bbsink *sink, const char *filename,
111 : : const char *linktarget, struct stat *statbuf,
112 : : bool sizeonly);
113 : : static void _tarWritePadding(bbsink *sink, int len);
114 : : static void convert_link_to_directory(const char *pathbuf, struct stat *statbuf);
115 : : static void perform_base_backup(basebackup_options *opt, bbsink *sink,
116 : : IncrementalBackupInfo *ib);
117 : : static void parse_basebackup_options(List *options, basebackup_options *opt);
118 : : static int compareWalFileNames(const ListCell *a, const ListCell *b);
119 : : static ssize_t basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset,
120 : : const char *filename, bool partial_read_ok);
121 : :
122 : : /* Was the backup currently in-progress initiated in recovery mode? */
123 : : static bool backup_started_in_recovery = false;
124 : :
125 : : /* Total number of checksum failures during base backup. */
126 : : static long long int total_checksum_failures;
127 : :
128 : : /* Do not verify checksums. */
129 : : static bool noverify_checksums = false;
130 : :
131 : : /*
132 : : * Definition of one element part of an exclusion list, used for paths part
133 : : * of checksum validation or base backups. "name" is the name of the file
134 : : * or path to check for exclusion. If "match_prefix" is true, any items
135 : : * matching the name as prefix are excluded.
136 : : */
137 : : struct exclude_list_item
138 : : {
139 : : const char *name;
140 : : bool match_prefix;
141 : : };
142 : :
143 : : /*
144 : : * The contents of these directories are removed or recreated during server
145 : : * start so they are not included in backups. The directories themselves are
146 : : * kept and included as empty to preserve access permissions.
147 : : *
148 : : * Note: this list should be kept in sync with the filter lists in pg_rewind's
149 : : * filemap.c.
150 : : */
151 : : static const char *const excludeDirContents[] =
152 : : {
153 : : /*
154 : : * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped
155 : : * because extensions like pg_stat_statements store data there.
156 : : */
157 : : PG_STAT_TMP_DIR,
158 : :
159 : : /*
160 : : * It is generally not useful to backup the contents of this directory
161 : : * even if the intention is to restore to another primary. See backup.sgml
162 : : * for a more detailed description.
163 : : */
164 : : PG_REPLSLOT_DIR,
165 : :
166 : : /* Contents removed on startup, see dsm_cleanup_for_mmap(). */
167 : : PG_DYNSHMEM_DIR,
168 : :
169 : : /* Contents removed on startup, see AsyncShmemInit(). */
170 : : "pg_notify",
171 : :
172 : : /*
173 : : * Old contents are loaded for possible debugging but are not required for
174 : : * normal operation, see SerialInit().
175 : : */
176 : : "pg_serial",
177 : :
178 : : /* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */
179 : : "pg_snapshots",
180 : :
181 : : /* Contents zeroed on startup, see StartupSUBTRANS(). */
182 : : "pg_subtrans",
183 : :
184 : : /* end of list */
185 : : NULL
186 : : };
187 : :
188 : : /*
189 : : * List of files excluded from backups.
190 : : */
191 : : static const struct exclude_list_item excludeFiles[] =
192 : : {
193 : : /* Skip auto conf temporary file. */
194 : : {PG_AUTOCONF_FILENAME ".tmp", false},
195 : :
196 : : /* Skip current log file temporary file */
197 : : {LOG_METAINFO_DATAFILE_TMP, false},
198 : :
199 : : /*
200 : : * Skip relation cache because it is rebuilt on startup. This includes
201 : : * temporary files.
202 : : */
203 : : {RELCACHE_INIT_FILENAME, true},
204 : :
205 : : /*
206 : : * backup_label and tablespace_map should not exist in a running cluster
207 : : * capable of doing an online backup, but exclude them just in case.
208 : : */
209 : : {BACKUP_LABEL_FILE, false},
210 : : {TABLESPACE_MAP, false},
211 : :
212 : : /*
213 : : * If there's a backup_manifest, it belongs to a backup that was used to
214 : : * start this server. It is *not* correct for this backup. Our
215 : : * backup_manifest is injected into the backup separately if users want
216 : : * it.
217 : : */
218 : : {"backup_manifest", false},
219 : :
220 : : {"postmaster.pid", false},
221 : : {"postmaster.opts", false},
222 : :
223 : : /* end of list */
224 : : {NULL, false}
225 : : };
226 : :
227 : : /*
228 : : * Actually do a base backup for the specified tablespaces.
229 : : *
230 : : * This is split out mainly to avoid complaints about "variable might be
231 : : * clobbered by longjmp" from stupider versions of gcc.
232 : : */
233 : : static void
626 rhaas@postgresql.org 234 :CBC 164 : perform_base_backup(basebackup_options *opt, bbsink *sink,
235 : : IncrementalBackupInfo *ib)
236 : : {
237 : : bbsink_state state;
238 : : XLogRecPtr endptr;
239 : : TimeLineID endtli;
240 : : backup_manifest_info manifest;
241 : : BackupState *backup_state;
242 : : StringInfo tablespace_map;
243 : :
244 : : /* Initial backup state, insofar as we know it now. */
1401 245 : 164 : state.tablespaces = NIL;
246 : 164 : state.tablespace_num = 0;
247 : 164 : state.bytes_done = 0;
248 : 164 : state.bytes_total = 0;
249 : 164 : state.bytes_total_is_valid = false;
250 : :
251 : : /* we're going to use a BufFile, so we need a ResourceOwner */
333 andres@anarazel.de 252 [ - + ]: 164 : Assert(AuxProcessResourceOwner != NULL);
253 [ + + - + ]: 164 : Assert(CurrentResourceOwner == AuxProcessResourceOwner ||
254 : : CurrentResourceOwner == NULL);
255 : 164 : CurrentResourceOwner = AuxProcessResourceOwner;
256 : :
4650 heikki.linnakangas@i 257 : 164 : backup_started_in_recovery = RecoveryInProgress();
258 : :
1962 rhaas@postgresql.org 259 : 164 : InitializeBackupManifest(&manifest, opt->manifest,
260 : : opt->manifest_checksum_type);
261 : :
2713 magnus@hagander.net 262 : 164 : total_checksum_failures = 0;
263 : :
264 : : /* Allocate backup related variables. */
1076 michael@paquier.xyz 265 : 164 : backup_state = (BackupState *) palloc0(sizeof(BackupState));
266 : 164 : tablespace_map = makeStringInfo();
267 : :
1401 rhaas@postgresql.org 268 : 164 : basebackup_progress_wait_checkpoint();
1076 michael@paquier.xyz 269 : 164 : do_pg_backup_start(opt->label, opt->fastcheckpoint, &state.tablespaces,
270 : : backup_state, tablespace_map);
271 : :
272 : 164 : state.startptr = backup_state->startpoint;
273 : 164 : state.starttli = backup_state->starttli;
274 : :
275 : : /*
276 : : * Once do_pg_backup_start has been called, ensure that any failure causes
277 : : * us to abort the backup so we don't "leak" a backup counter. For this
278 : : * reason, *all* functionality between do_pg_backup_start() and the end of
279 : : * do_pg_backup_stop() should be inside the error cleanup block!
280 : : */
281 : :
2088 rhaas@postgresql.org 282 [ + + ]: 164 : PG_ENSURE_ERROR_CLEANUP(do_pg_abort_backup, BoolGetDatum(false));
283 : : {
284 : : ListCell *lc;
285 : : tablespaceinfo *newti;
286 : :
287 : : /* If this is an incremental backup, execute preparatory steps. */
626 288 [ + + ]: 164 : if (ib != NULL)
289 : 10 : PrepareForIncrementalBackup(ib, backup_state);
290 : :
291 : : /* Add a node for the base directory at the end */
1067 drowley@postgresql.o 292 : 164 : newti = palloc0(sizeof(tablespaceinfo));
293 : 164 : newti->size = -1;
294 : 164 : state.tablespaces = lappend(state.tablespaces, newti);
295 : :
296 : : /*
297 : : * Calculate the total backup size by summing up the size of each
298 : : * tablespace
299 : : */
2013 fujii@postgresql.org 300 [ + - ]: 164 : if (opt->progress)
301 : : {
1401 rhaas@postgresql.org 302 : 164 : basebackup_progress_estimate_backup_size();
303 : :
304 [ + - + + : 367 : foreach(lc, state.tablespaces)
+ + ]
305 : : {
2013 fujii@postgresql.org 306 : 203 : tablespaceinfo *tmp = (tablespaceinfo *) lfirst(lc);
307 : :
1907 rhaas@postgresql.org 308 [ + + ]: 203 : if (tmp->path == NULL)
1401 309 : 164 : tmp->size = sendDir(sink, ".", 1, true, state.tablespaces,
310 : : true, NULL, InvalidOid, NULL);
311 : : else
312 : 39 : tmp->size = sendTablespace(sink, tmp->path, tmp->oid, true,
313 : : NULL, NULL);
314 : 203 : state.bytes_total += tmp->size;
315 : : }
316 : 164 : state.bytes_total_is_valid = true;
317 : : }
318 : :
319 : : /* notify basebackup sink about start of backup */
320 : 164 : bbsink_begin_backup(sink, &state, SINK_BUFFER_LENGTH);
321 : :
322 : : /* Send off our tablespaces one by one */
323 [ + - + + : 362 : foreach(lc, state.tablespaces)
+ + ]
324 : : {
5352 tgl@sss.pgh.pa.us 325 : 203 : tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
326 : :
4973 simon@2ndQuadrant.co 327 [ + + ]: 203 : if (ti->path == NULL)
328 : : {
329 : : struct stat statbuf;
1578 tgl@sss.pgh.pa.us 330 : 164 : bool sendtblspclinks = true;
331 : : char *backup_label;
332 : :
1401 rhaas@postgresql.org 333 : 164 : bbsink_begin_archive(sink, "base.tar");
334 : :
335 : : /* In the main tar, include the backup_label first... */
1076 michael@paquier.xyz 336 : 164 : backup_label = build_backup_content(backup_state, false);
337 : 164 : sendFileWithContent(sink, BACKUP_LABEL_FILE,
338 : : backup_label, -1, &manifest);
339 : 164 : pfree(backup_label);
340 : :
341 : : /* Then the tablespace_map file, if required... */
1907 rhaas@postgresql.org 342 [ + + ]: 164 : if (opt->sendtblspcmapfile)
343 : : {
1076 michael@paquier.xyz 344 : 32 : sendFileWithContent(sink, TABLESPACE_MAP,
663 345 : 32 : tablespace_map->data, -1, &manifest);
1907 rhaas@postgresql.org 346 : 32 : sendtblspclinks = false;
347 : : }
348 : :
349 : : /* Then the bulk of the files... */
1401 350 : 164 : sendDir(sink, ".", 1, false, state.tablespaces,
351 : : sendtblspclinks, &manifest, InvalidOid, ib);
352 : :
353 : : /* ... and pg_control after everything else. */
4973 simon@2ndQuadrant.co 354 [ - + ]: 159 : if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
4973 simon@2ndQuadrant.co 355 [ # # ]:UBC 0 : ereport(ERROR,
356 : : (errcode_for_file_access(),
357 : : errmsg("could not stat file \"%s\": %m",
358 : : XLOG_CONTROL_FILE)));
1401 rhaas@postgresql.org 359 :CBC 159 : sendFile(sink, XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf,
360 : : false, InvalidOid, InvalidOid,
361 : : InvalidRelFileNumber, 0, &manifest, 0, NULL, 0);
362 : : }
363 : : else
364 : : {
684 365 : 39 : char *archive_name = psprintf("%u.tar", ti->oid);
366 : :
1401 367 : 39 : bbsink_begin_archive(sink, archive_name);
368 : :
626 369 : 39 : sendTablespace(sink, ti->path, ti->oid, false, &manifest, ib);
370 : : }
371 : :
372 : : /*
373 : : * If we're including WAL, and this is the main data directory we
374 : : * don't treat this as the end of the tablespace. Instead, we will
375 : : * include the xlog files below and stop afterwards. This is safe
376 : : * since the main data directory is always sent *last*.
377 : : */
5333 magnus@hagander.net 378 [ + + + + ]: 198 : if (opt->includewal && ti->path == NULL)
379 : : {
1401 rhaas@postgresql.org 380 [ - + ]: 22 : Assert(lnext(state.tablespaces, lc) == NULL);
381 : : }
382 : : else
383 : : {
384 : : /* Properly terminate the tarfile. */
385 : : StaticAssertDecl(2 * TAR_BLOCK_SIZE <= BLCKSZ,
386 : : "BLCKSZ too small for 2 tar blocks");
1397 387 : 176 : memset(sink->bbs_buffer, 0, 2 * TAR_BLOCK_SIZE);
388 : 176 : bbsink_archive_contents(sink, 2 * TAR_BLOCK_SIZE);
389 : :
390 : : /* OK, that's the end of the archive. */
1401 391 : 176 : bbsink_end_archive(sink);
392 : : }
393 : : }
394 : :
395 : 159 : basebackup_progress_wait_wal_archive(&state);
1076 michael@paquier.xyz 396 : 159 : do_pg_backup_stop(backup_state, !opt->nowait);
397 : :
398 : 159 : endptr = backup_state->stoppoint;
399 : 159 : endtli = backup_state->stoptli;
400 : :
401 : : /* Deallocate backup-related variables. */
539 dgustafsson@postgres 402 : 159 : destroyStringInfo(tablespace_map);
1076 michael@paquier.xyz 403 : 159 : pfree(backup_state);
404 : : }
2088 rhaas@postgresql.org 405 [ - + ]: 160 : PG_END_ENSURE_ERROR_CLEANUP(do_pg_abort_backup, BoolGetDatum(false));
406 : :
407 : :
5333 magnus@hagander.net 408 [ + + ]: 159 : if (opt->includewal)
409 : : {
410 : : /*
411 : : * We've left the last tar file "open", so we can now append the
412 : : * required WAL files to it.
413 : : */
414 : : char pathbuf[MAXPGPATH];
415 : : XLogSegNo segno;
416 : : XLogSegNo startsegno;
417 : : XLogSegNo endsegno;
418 : : struct stat statbuf;
4629 heikki.linnakangas@i 419 : 22 : List *historyFileList = NIL;
420 : 22 : List *walFileList = NIL;
421 : : char firstoff[MAXFNAMELEN];
422 : : char lastoff[MAXFNAMELEN];
423 : : DIR *dir;
424 : : struct dirent *de;
425 : : ListCell *lc;
426 : : TimeLineID tli;
427 : :
1401 rhaas@postgresql.org 428 : 22 : basebackup_progress_transfer_wal();
429 : :
430 : : /*
431 : : * I'd rather not worry about timelines here, so scan pg_wal and
432 : : * include all WAL files in the range between 'startptr' and 'endptr',
433 : : * regardless of the timeline the file is stamped with. If there are
434 : : * some spurious WAL files belonging to timelines that don't belong in
435 : : * this server's history, they will be included too. Normally there
436 : : * shouldn't be such files, but if there are, there's little harm in
437 : : * including them.
438 : : */
439 : 22 : XLByteToSeg(state.startptr, startsegno, wal_segment_size);
440 : 22 : XLogFileName(firstoff, state.starttli, startsegno, wal_segment_size);
2909 andres@anarazel.de 441 : 22 : XLByteToPrevSeg(endptr, endsegno, wal_segment_size);
1408 rhaas@postgresql.org 442 : 22 : XLogFileName(lastoff, endtli, endsegno, wal_segment_size);
443 : :
3243 444 : 22 : dir = AllocateDir("pg_wal");
445 [ + + ]: 157 : while ((de = ReadDir(dir, "pg_wal")) != NULL)
446 : : {
447 : : /* Does it look like a WAL segment, and is it in the range? */
3774 heikki.linnakangas@i 448 [ + + ]: 135 : if (IsXLogFileName(de->d_name) &&
4629 449 [ + - ]: 47 : strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
450 [ + + ]: 47 : strcmp(de->d_name + 8, lastoff + 8) <= 0)
451 : : {
452 : 22 : walFileList = lappend(walFileList, pstrdup(de->d_name));
453 : : }
454 : : /* Does it look like a timeline history file? */
3774 455 [ - + ]: 113 : else if (IsTLHistoryFileName(de->d_name))
456 : : {
4629 heikki.linnakangas@i 457 :UBC 0 : historyFileList = lappend(historyFileList, pstrdup(de->d_name));
458 : : }
459 : : }
4629 heikki.linnakangas@i 460 :CBC 22 : FreeDir(dir);
461 : :
462 : : /*
463 : : * Before we go any further, check that none of the WAL segments we
464 : : * need were removed.
465 : : */
1401 rhaas@postgresql.org 466 : 22 : CheckXLogRemoved(startsegno, state.starttli);
467 : :
468 : : /*
469 : : * Sort the WAL filenames. We want to send the files in order from
470 : : * oldest to newest, to reduce the chance that a file is recycled
471 : : * before we get a chance to send it over.
472 : : */
2244 tgl@sss.pgh.pa.us 473 : 22 : list_sort(walFileList, compareWalFileNames);
474 : :
475 : : /*
476 : : * There must be at least one xlog file in the pg_wal directory, since
477 : : * we are doing backup-including-xlog.
478 : : */
479 [ - + ]: 22 : if (walFileList == NIL)
4396 magnus@hagander.net 480 [ # # ]:UBC 0 : ereport(ERROR,
481 : : (errmsg("could not find any WAL files")));
482 : :
483 : : /*
484 : : * Sanity check: the first and last segment should cover startptr and
485 : : * endptr, with no gaps in between.
486 : : */
2244 tgl@sss.pgh.pa.us 487 :CBC 22 : XLogFromFileName((char *) linitial(walFileList),
488 : : &tli, &segno, wal_segment_size);
4629 heikki.linnakangas@i 489 [ - + ]: 22 : if (segno != startsegno)
490 : : {
491 : : char startfname[MAXFNAMELEN];
492 : :
1401 rhaas@postgresql.org 493 :UBC 0 : XLogFileName(startfname, state.starttli, startsegno,
494 : : wal_segment_size);
4629 heikki.linnakangas@i 495 [ # # ]: 0 : ereport(ERROR,
496 : : (errmsg("could not find WAL file \"%s\"", startfname)));
497 : : }
2244 tgl@sss.pgh.pa.us 498 [ + - + + :CBC 44 : foreach(lc, walFileList)
+ + ]
499 : : {
500 : 22 : char *walFileName = (char *) lfirst(lc);
4483 bruce@momjian.us 501 : 22 : XLogSegNo currsegno = segno;
502 : 22 : XLogSegNo nextsegno = segno + 1;
503 : :
2244 tgl@sss.pgh.pa.us 504 : 22 : XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
4629 heikki.linnakangas@i 505 [ + - - + ]: 22 : if (!(nextsegno == segno || currsegno == segno))
506 : : {
507 : : char nextfname[MAXFNAMELEN];
508 : :
1408 rhaas@postgresql.org 509 :UBC 0 : XLogFileName(nextfname, tli, nextsegno, wal_segment_size);
4629 heikki.linnakangas@i 510 [ # # ]: 0 : ereport(ERROR,
511 : : (errmsg("could not find WAL file \"%s\"", nextfname)));
512 : : }
513 : : }
4629 heikki.linnakangas@i 514 [ - + ]:CBC 22 : if (segno != endsegno)
515 : : {
516 : : char endfname[MAXFNAMELEN];
517 : :
1408 rhaas@postgresql.org 518 :UBC 0 : XLogFileName(endfname, endtli, endsegno, wal_segment_size);
4629 heikki.linnakangas@i 519 [ # # ]: 0 : ereport(ERROR,
520 : : (errmsg("could not find WAL file \"%s\"", endfname)));
521 : : }
522 : :
523 : : /* Ok, we have everything we need. Send the WAL files. */
2244 tgl@sss.pgh.pa.us 524 [ + - + + :CBC 44 : foreach(lc, walFileList)
+ + ]
525 : : {
526 : 22 : char *walFileName = (char *) lfirst(lc);
527 : : int fd;
528 : : ssize_t cnt;
4629 heikki.linnakangas@i 529 : 22 : pgoff_t len = 0;
530 : :
2244 tgl@sss.pgh.pa.us 531 : 22 : snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFileName);
532 : 22 : XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
533 : :
1907 rhaas@postgresql.org 534 : 22 : fd = OpenTransientFile(pathbuf, O_RDONLY | PG_BINARY);
535 [ - + ]: 22 : if (fd < 0)
536 : : {
2630 michael@paquier.xyz 537 :UBC 0 : int save_errno = errno;
538 : :
539 : : /*
540 : : * Most likely reason for this is that the file was already
541 : : * removed by a checkpoint, so check for that to get a better
542 : : * error message.
543 : : */
4629 heikki.linnakangas@i 544 : 0 : CheckXLogRemoved(segno, tli);
545 : :
2630 michael@paquier.xyz 546 : 0 : errno = save_errno;
4629 heikki.linnakangas@i 547 [ # # ]: 0 : ereport(ERROR,
548 : : (errcode_for_file_access(),
549 : : errmsg("could not open file \"%s\": %m", pathbuf)));
550 : : }
551 : :
1907 rhaas@postgresql.org 552 [ - + ]:CBC 22 : if (fstat(fd, &statbuf) != 0)
4629 heikki.linnakangas@i 553 [ # # ]:UBC 0 : ereport(ERROR,
554 : : (errcode_for_file_access(),
555 : : errmsg("could not stat file \"%s\": %m",
556 : : pathbuf)));
2909 andres@anarazel.de 557 [ - + ]:CBC 22 : if (statbuf.st_size != wal_segment_size)
558 : : {
4629 heikki.linnakangas@i 559 :UBC 0 : CheckXLogRemoved(segno, tli);
560 [ # # ]: 0 : ereport(ERROR,
561 : : (errcode_for_file_access(),
562 : : errmsg("unexpected WAL file size \"%s\"", walFileName)));
563 : : }
564 : :
565 : : /* send the WAL file itself */
1401 rhaas@postgresql.org 566 :CBC 22 : _tarWriteHeader(sink, pathbuf, NULL, &statbuf, false);
567 : :
568 : 22 : while ((cnt = basebackup_read_file(fd, sink->bbs_buffer,
569 : 11264 : Min(sink->bbs_buffer_length,
570 : : wal_segment_size - len),
1907 571 [ + - ]: 11264 : len, pathbuf, true)) > 0)
572 : : {
4629 heikki.linnakangas@i 573 : 11264 : CheckXLogRemoved(segno, tli);
1401 rhaas@postgresql.org 574 : 11264 : bbsink_archive_contents(sink, cnt);
575 : :
4629 heikki.linnakangas@i 576 : 11264 : len += cnt;
577 : :
2909 andres@anarazel.de 578 [ + + ]: 11264 : if (len == wal_segment_size)
4629 heikki.linnakangas@i 579 : 22 : break;
580 : : }
581 : :
2909 andres@anarazel.de 582 [ - + ]: 22 : if (len != wal_segment_size)
583 : : {
4629 heikki.linnakangas@i 584 :UBC 0 : CheckXLogRemoved(segno, tli);
585 [ # # ]: 0 : ereport(ERROR,
586 : : (errcode_for_file_access(),
587 : : errmsg("unexpected WAL file size \"%s\"", walFileName)));
588 : : }
589 : :
590 : : /*
591 : : * wal_segment_size is a multiple of TAR_BLOCK_SIZE, so no need
592 : : * for padding.
593 : : */
1961 rhaas@postgresql.org 594 [ - + ]:CBC 22 : Assert(wal_segment_size % TAR_BLOCK_SIZE == 0);
595 : :
1907 596 : 22 : CloseTransientFile(fd);
597 : :
598 : : /*
599 : : * Mark file as archived, otherwise files can get archived again
600 : : * after promotion of a new node. This is in line with
601 : : * walreceiver.c always doing an XLogArchiveForceDone() after a
602 : : * complete segment.
603 : : */
2244 tgl@sss.pgh.pa.us 604 : 22 : StatusFilePath(pathbuf, walFileName, ".done");
663 michael@paquier.xyz 605 : 22 : sendFileWithContent(sink, pathbuf, "", -1, &manifest);
606 : : }
607 : :
608 : : /*
609 : : * Send timeline history files too. Only the latest timeline history
610 : : * file is required for recovery, and even that only if there happens
611 : : * to be a timeline switch in the first WAL segment that contains the
612 : : * checkpoint record, or if we're taking a base backup from a standby
613 : : * server and the target timeline changes while the backup is taken.
614 : : * But they are small and highly useful for debugging purposes, so
615 : : * better include them all, always.
616 : : */
4629 heikki.linnakangas@i 617 [ - + - - : 22 : foreach(lc, historyFileList)
- + ]
618 : : {
4483 bruce@momjian.us 619 :UBC 0 : char *fname = lfirst(lc);
620 : :
4629 heikki.linnakangas@i 621 : 0 : snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
622 : :
623 [ # # ]: 0 : if (lstat(pathbuf, &statbuf) != 0)
624 [ # # ]: 0 : ereport(ERROR,
625 : : (errcode_for_file_access(),
626 : : errmsg("could not stat file \"%s\": %m", pathbuf)));
627 : :
684 rhaas@postgresql.org 628 : 0 : sendFile(sink, pathbuf, pathbuf, &statbuf, false,
629 : : InvalidOid, InvalidOid, InvalidRelFileNumber, 0,
630 : : &manifest, 0, NULL, 0);
631 : :
632 : : /* unconditionally mark file as archived */
3899 andres@anarazel.de 633 : 0 : StatusFilePath(pathbuf, fname, ".done");
663 michael@paquier.xyz 634 : 0 : sendFileWithContent(sink, pathbuf, "", -1, &manifest);
635 : : }
636 : :
637 : : /* Properly terminate the tar file. */
638 : : StaticAssertStmt(2 * TAR_BLOCK_SIZE <= BLCKSZ,
639 : : "BLCKSZ too small for 2 tar blocks");
1397 rhaas@postgresql.org 640 :CBC 22 : memset(sink->bbs_buffer, 0, 2 * TAR_BLOCK_SIZE);
641 : 22 : bbsink_archive_contents(sink, 2 * TAR_BLOCK_SIZE);
642 : :
643 : : /* OK, that's the end of the archive. */
1401 644 : 22 : bbsink_end_archive(sink);
645 : : }
646 : :
647 : 159 : AddWALInfoToBackupManifest(&manifest, state.startptr, state.starttli,
648 : : endptr, endtli);
649 : :
650 : 159 : SendBackupManifest(&manifest, sink);
651 : :
652 : 159 : bbsink_end_backup(sink, endptr, endtli);
653 : :
2713 magnus@hagander.net 654 [ + + ]: 159 : if (total_checksum_failures)
655 : : {
656 [ + + ]: 3 : if (total_checksum_failures > 1)
657 [ + - ]: 2 : ereport(WARNING,
658 : : (errmsg_plural("%lld total checksum verification failure",
659 : : "%lld total checksum verification failures",
660 : : total_checksum_failures,
661 : : total_checksum_failures)));
662 : :
663 [ + - ]: 3 : ereport(ERROR,
664 : : (errcode(ERRCODE_DATA_CORRUPTED),
665 : : errmsg("checksum verification failure during base backup")));
666 : : }
667 : :
668 : : /*
669 : : * Make sure to free the manifest before the resource owners as manifests
670 : : * use cryptohash contexts that may depend on resource owners (like
671 : : * OpenSSL).
672 : : */
1737 michael@paquier.xyz 673 : 156 : FreeBackupManifest(&manifest);
674 : :
675 : : /* clean up the resource owner we created */
333 andres@anarazel.de 676 : 156 : ReleaseAuxProcessResources(true);
677 : :
1401 rhaas@postgresql.org 678 : 156 : basebackup_progress_done();
5352 tgl@sss.pgh.pa.us 679 : 156 : }
680 : :
681 : : /*
682 : : * list_sort comparison function, to compare log/seg portion of WAL segment
683 : : * filenames, ignoring the timeline portion.
684 : : */
685 : : static int
2244 tgl@sss.pgh.pa.us 686 :UBC 0 : compareWalFileNames(const ListCell *a, const ListCell *b)
687 : : {
688 : 0 : char *fna = (char *) lfirst(a);
689 : 0 : char *fnb = (char *) lfirst(b);
690 : :
4629 heikki.linnakangas@i 691 : 0 : return strcmp(fna + 8, fnb + 8);
692 : : }
693 : :
694 : : /*
695 : : * Parse the base backup options passed down by the parser
696 : : */
697 : : static void
5340 magnus@hagander.net 698 :CBC 181 : parse_basebackup_options(List *options, basebackup_options *opt)
699 : : {
700 : : ListCell *lopt;
701 : 181 : bool o_label = false;
702 : 181 : bool o_progress = false;
1432 rhaas@postgresql.org 703 : 181 : bool o_checkpoint = false;
5323 magnus@hagander.net 704 : 181 : bool o_nowait = false;
5333 705 : 181 : bool o_wal = false;
626 rhaas@postgresql.org 706 : 181 : bool o_incremental = false;
4209 alvherre@alvh.no-ip. 707 : 181 : bool o_maxrate = false;
3770 andrew@dunslane.net 708 : 181 : bool o_tablespace_map = false;
2713 magnus@hagander.net 709 : 181 : bool o_noverify_checksums = false;
1982 rhaas@postgresql.org 710 : 181 : bool o_manifest = false;
711 : 181 : bool o_manifest_checksums = false;
1327 712 : 181 : bool o_target = false;
1390 713 : 181 : bool o_target_detail = false;
1271 714 : 181 : char *target_str = NULL;
715 : 181 : char *target_detail_str = NULL;
1321 716 : 181 : bool o_compression = false;
1263 717 : 181 : bool o_compression_detail = false;
718 : 181 : char *compression_detail_str = NULL;
719 : :
5338 magnus@hagander.net 720 [ + - + - : 1991 : MemSet(opt, 0, sizeof(*opt));
+ - + - +
+ ]
1982 rhaas@postgresql.org 721 : 181 : opt->manifest = MANIFEST_OPTION_NO;
722 : 181 : opt->manifest_checksum_type = CHECKSUM_TYPE_CRC32C;
1243 michael@paquier.xyz 723 : 181 : opt->compression = PG_COMPRESSION_NONE;
724 : 181 : opt->compression_specification.algorithm = PG_COMPRESSION_NONE;
725 : :
5340 magnus@hagander.net 726 [ + - + + : 1387 : foreach(lopt, options)
+ + ]
727 : : {
728 : 1209 : DefElem *defel = (DefElem *) lfirst(lopt);
729 : :
730 [ + + ]: 1209 : if (strcmp(defel->defname, "label") == 0)
731 : : {
732 [ - + ]: 181 : if (o_label)
5340 magnus@hagander.net 733 [ # # ]:UBC 0 : ereport(ERROR,
734 : : (errcode(ERRCODE_SYNTAX_ERROR),
735 : : errmsg("duplicate option \"%s\"", defel->defname)));
1432 rhaas@postgresql.org 736 :CBC 181 : opt->label = defGetString(defel);
5340 magnus@hagander.net 737 : 181 : o_label = true;
738 : : }
739 [ + + ]: 1028 : else if (strcmp(defel->defname, "progress") == 0)
740 : : {
741 [ - + ]: 181 : if (o_progress)
5340 magnus@hagander.net 742 [ # # ]:UBC 0 : ereport(ERROR,
743 : : (errcode(ERRCODE_SYNTAX_ERROR),
744 : : errmsg("duplicate option \"%s\"", defel->defname)));
1432 rhaas@postgresql.org 745 :CBC 181 : opt->progress = defGetBoolean(defel);
5340 magnus@hagander.net 746 : 181 : o_progress = true;
747 : : }
1432 rhaas@postgresql.org 748 [ + + ]: 847 : else if (strcmp(defel->defname, "checkpoint") == 0)
749 : : {
750 : 171 : char *optval = defGetString(defel);
751 : :
752 [ - + ]: 171 : if (o_checkpoint)
5340 magnus@hagander.net 753 [ # # ]:UBC 0 : ereport(ERROR,
754 : : (errcode(ERRCODE_SYNTAX_ERROR),
755 : : errmsg("duplicate option \"%s\"", defel->defname)));
1432 rhaas@postgresql.org 756 [ + - ]:CBC 171 : if (pg_strcasecmp(optval, "fast") == 0)
757 : 171 : opt->fastcheckpoint = true;
1432 rhaas@postgresql.org 758 [ # # ]:UBC 0 : else if (pg_strcasecmp(optval, "spread") == 0)
759 : 0 : opt->fastcheckpoint = false;
760 : : else
761 [ # # ]: 0 : ereport(ERROR,
762 : : (errcode(ERRCODE_SYNTAX_ERROR),
763 : : errmsg("unrecognized checkpoint type: \"%s\"",
764 : : optval)));
1432 rhaas@postgresql.org 765 :CBC 171 : o_checkpoint = true;
766 : : }
767 [ + + ]: 676 : else if (strcmp(defel->defname, "wait") == 0)
768 : : {
5323 magnus@hagander.net 769 [ - + ]: 172 : if (o_nowait)
5323 magnus@hagander.net 770 [ # # ]:UBC 0 : ereport(ERROR,
771 : : (errcode(ERRCODE_SYNTAX_ERROR),
772 : : errmsg("duplicate option \"%s\"", defel->defname)));
1432 rhaas@postgresql.org 773 :CBC 172 : opt->nowait = !defGetBoolean(defel);
5323 magnus@hagander.net 774 : 172 : o_nowait = true;
775 : : }
5333 776 [ + + ]: 504 : else if (strcmp(defel->defname, "wal") == 0)
777 : : {
778 [ - + ]: 26 : if (o_wal)
5333 magnus@hagander.net 779 [ # # ]:UBC 0 : ereport(ERROR,
780 : : (errcode(ERRCODE_SYNTAX_ERROR),
781 : : errmsg("duplicate option \"%s\"", defel->defname)));
1432 rhaas@postgresql.org 782 :CBC 26 : opt->includewal = defGetBoolean(defel);
5333 magnus@hagander.net 783 : 26 : o_wal = true;
784 : : }
626 rhaas@postgresql.org 785 [ + + ]: 478 : else if (strcmp(defel->defname, "incremental") == 0)
786 : : {
787 [ - + ]: 10 : if (o_incremental)
626 rhaas@postgresql.org 788 [ # # ]:UBC 0 : ereport(ERROR,
789 : : (errcode(ERRCODE_SYNTAX_ERROR),
790 : : errmsg("duplicate option \"%s\"", defel->defname)));
626 rhaas@postgresql.org 791 :CBC 10 : opt->incremental = defGetBoolean(defel);
792 [ + - - + ]: 10 : if (opt->incremental && !summarize_wal)
626 rhaas@postgresql.org 793 [ # # ]:UBC 0 : ereport(ERROR,
794 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
795 : : errmsg("incremental backups cannot be taken unless WAL summarization is enabled")));
626 rhaas@postgresql.org 796 :CBC 10 : o_incremental = true;
797 : : }
4209 alvherre@alvh.no-ip. 798 [ + + ]: 468 : else if (strcmp(defel->defname, "max_rate") == 0)
799 : : {
800 : : int64 maxrate;
801 : :
802 [ - + ]: 1 : if (o_maxrate)
4209 alvherre@alvh.no-ip. 803 [ # # ]:UBC 0 : ereport(ERROR,
804 : : (errcode(ERRCODE_SYNTAX_ERROR),
805 : : errmsg("duplicate option \"%s\"", defel->defname)));
806 : :
1432 rhaas@postgresql.org 807 :CBC 1 : maxrate = defGetInt64(defel);
4209 alvherre@alvh.no-ip. 808 [ + - - + ]: 1 : if (maxrate < MAX_RATE_LOWER || maxrate > MAX_RATE_UPPER)
4209 alvherre@alvh.no-ip. 809 [ # # ]:UBC 0 : ereport(ERROR,
810 : : (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
811 : : errmsg("%d is outside the valid range for parameter \"%s\" (%d .. %d)",
812 : : (int) maxrate, "MAX_RATE", MAX_RATE_LOWER, MAX_RATE_UPPER)));
813 : :
4209 alvherre@alvh.no-ip. 814 :CBC 1 : opt->maxrate = (uint32) maxrate;
815 : 1 : o_maxrate = true;
816 : : }
3770 andrew@dunslane.net 817 [ + + ]: 467 : else if (strcmp(defel->defname, "tablespace_map") == 0)
818 : : {
819 [ - + ]: 38 : if (o_tablespace_map)
3770 andrew@dunslane.net 820 [ # # ]:UBC 0 : ereport(ERROR,
821 : : (errcode(ERRCODE_SYNTAX_ERROR),
822 : : errmsg("duplicate option \"%s\"", defel->defname)));
1432 rhaas@postgresql.org 823 :CBC 38 : opt->sendtblspcmapfile = defGetBoolean(defel);
3770 andrew@dunslane.net 824 : 38 : o_tablespace_map = true;
825 : : }
1432 rhaas@postgresql.org 826 [ + + ]: 429 : else if (strcmp(defel->defname, "verify_checksums") == 0)
827 : : {
2713 magnus@hagander.net 828 [ - + ]: 1 : if (o_noverify_checksums)
2713 magnus@hagander.net 829 [ # # ]:UBC 0 : ereport(ERROR,
830 : : (errcode(ERRCODE_SYNTAX_ERROR),
831 : : errmsg("duplicate option \"%s\"", defel->defname)));
1432 rhaas@postgresql.org 832 :CBC 1 : noverify_checksums = !defGetBoolean(defel);
2713 magnus@hagander.net 833 : 1 : o_noverify_checksums = true;
834 : : }
1982 rhaas@postgresql.org 835 [ + + ]: 428 : else if (strcmp(defel->defname, "manifest") == 0)
836 : : {
1432 837 : 180 : char *optval = defGetString(defel);
838 : : bool manifest_bool;
839 : :
1982 840 [ - + ]: 180 : if (o_manifest)
1982 rhaas@postgresql.org 841 [ # # ]:UBC 0 : ereport(ERROR,
842 : : (errcode(ERRCODE_SYNTAX_ERROR),
843 : : errmsg("duplicate option \"%s\"", defel->defname)));
1982 rhaas@postgresql.org 844 [ + - ]:CBC 180 : if (parse_bool(optval, &manifest_bool))
845 : : {
846 [ + - ]: 180 : if (manifest_bool)
847 : 180 : opt->manifest = MANIFEST_OPTION_YES;
848 : : else
1982 rhaas@postgresql.org 849 :UBC 0 : opt->manifest = MANIFEST_OPTION_NO;
850 : : }
851 [ # # ]: 0 : else if (pg_strcasecmp(optval, "force-encode") == 0)
852 : 0 : opt->manifest = MANIFEST_OPTION_FORCE_ENCODE;
853 : : else
854 [ # # ]: 0 : ereport(ERROR,
855 : : (errcode(ERRCODE_SYNTAX_ERROR),
856 : : errmsg("unrecognized manifest option: \"%s\"",
857 : : optval)));
1982 rhaas@postgresql.org 858 :CBC 180 : o_manifest = true;
859 : : }
860 [ + + ]: 248 : else if (strcmp(defel->defname, "manifest_checksums") == 0)
861 : : {
1432 862 : 14 : char *optval = defGetString(defel);
863 : :
1982 864 [ - + ]: 14 : if (o_manifest_checksums)
1982 rhaas@postgresql.org 865 [ # # ]:UBC 0 : ereport(ERROR,
866 : : (errcode(ERRCODE_SYNTAX_ERROR),
867 : : errmsg("duplicate option \"%s\"", defel->defname)));
1982 rhaas@postgresql.org 868 [ + + ]:CBC 14 : if (!pg_checksum_parse_type(optval,
869 : : &opt->manifest_checksum_type))
870 [ + - ]: 2 : ereport(ERROR,
871 : : (errcode(ERRCODE_SYNTAX_ERROR),
872 : : errmsg("unrecognized checksum algorithm: \"%s\"",
873 : : optval)));
874 : 12 : o_manifest_checksums = true;
875 : : }
1327 876 [ + + ]: 234 : else if (strcmp(defel->defname, "target") == 0)
877 : : {
878 [ - + ]: 179 : if (o_target)
1327 rhaas@postgresql.org 879 [ # # ]:UBC 0 : ereport(ERROR,
880 : : (errcode(ERRCODE_SYNTAX_ERROR),
881 : : errmsg("duplicate option \"%s\"", defel->defname)));
1271 rhaas@postgresql.org 882 :CBC 179 : target_str = defGetString(defel);
1327 883 : 179 : o_target = true;
884 : : }
1390 885 [ + + ]: 55 : else if (strcmp(defel->defname, "target_detail") == 0)
886 : : {
887 : 10 : char *optval = defGetString(defel);
888 : :
889 [ - + ]: 10 : if (o_target_detail)
1390 rhaas@postgresql.org 890 [ # # ]:UBC 0 : ereport(ERROR,
891 : : (errcode(ERRCODE_SYNTAX_ERROR),
892 : : errmsg("duplicate option \"%s\"", defel->defname)));
1271 rhaas@postgresql.org 893 :CBC 10 : target_detail_str = optval;
1390 894 : 10 : o_target_detail = true;
895 : : }
1321 896 [ + + ]: 45 : else if (strcmp(defel->defname, "compression") == 0)
897 : : {
898 : 30 : char *optval = defGetString(defel);
899 : :
900 [ - + ]: 30 : if (o_compression)
1321 rhaas@postgresql.org 901 [ # # ]:UBC 0 : ereport(ERROR,
902 : : (errcode(ERRCODE_SYNTAX_ERROR),
903 : : errmsg("duplicate option \"%s\"", defel->defname)));
1243 michael@paquier.xyz 904 [ + + ]:CBC 30 : if (!parse_compress_algorithm(optval, &opt->compression))
1321 rhaas@postgresql.org 905 [ + - ]: 1 : ereport(ERROR,
906 : : (errcode(ERRCODE_SYNTAX_ERROR),
907 : : errmsg("unrecognized compression algorithm: \"%s\"",
908 : : optval)));
909 : 29 : o_compression = true;
910 : : }
1263 911 [ + - ]: 15 : else if (strcmp(defel->defname, "compression_detail") == 0)
912 : : {
913 [ - + ]: 15 : if (o_compression_detail)
1321 rhaas@postgresql.org 914 [ # # ]:UBC 0 : ereport(ERROR,
915 : : (errcode(ERRCODE_SYNTAX_ERROR),
916 : : errmsg("duplicate option \"%s\"", defel->defname)));
1263 rhaas@postgresql.org 917 :CBC 15 : compression_detail_str = defGetString(defel);
918 : 15 : o_compression_detail = true;
919 : : }
920 : : else
1321 rhaas@postgresql.org 921 [ # # ]:UBC 0 : ereport(ERROR,
922 : : (errcode(ERRCODE_SYNTAX_ERROR),
923 : : errmsg("unrecognized base backup option: \"%s\"",
924 : : defel->defname)));
925 : : }
926 : :
5340 magnus@hagander.net 927 [ - + ]:CBC 178 : if (opt->label == NULL)
5340 magnus@hagander.net 928 :UBC 0 : opt->label = "base backup";
1982 rhaas@postgresql.org 929 [ + + ]:CBC 178 : if (opt->manifest == MANIFEST_OPTION_NO)
930 : : {
931 [ - + ]: 1 : if (o_manifest_checksums)
1982 rhaas@postgresql.org 932 [ # # ]:UBC 0 : ereport(ERROR,
933 : : (errcode(ERRCODE_SYNTAX_ERROR),
934 : : errmsg("manifest checksums require a backup manifest")));
1982 rhaas@postgresql.org 935 :CBC 1 : opt->manifest_checksum_type = CHECKSUM_TYPE_NONE;
936 : : }
937 : :
1271 938 [ - + ]: 178 : if (target_str == NULL)
939 : : {
1271 rhaas@postgresql.org 940 [ # # ]:UBC 0 : if (target_detail_str != NULL)
1390 941 [ # # ]: 0 : ereport(ERROR,
942 : : (errcode(ERRCODE_SYNTAX_ERROR),
943 : : errmsg("target detail cannot be used without target")));
1271 944 : 0 : opt->use_copytblspc = true;
945 : 0 : opt->send_to_client = true;
946 : : }
1271 rhaas@postgresql.org 947 [ + + ]:CBC 178 : else if (strcmp(target_str, "client") == 0)
948 : : {
949 [ - + ]: 162 : if (target_detail_str != NULL)
1390 rhaas@postgresql.org 950 [ # # ]:UBC 0 : ereport(ERROR,
951 : : (errcode(ERRCODE_SYNTAX_ERROR),
952 : : errmsg("target \"%s\" does not accept a target detail",
953 : : target_str)));
1271 rhaas@postgresql.org 954 :CBC 162 : opt->send_to_client = true;
955 : : }
956 : : else
957 : 14 : opt->target_handle =
958 : 16 : BaseBackupGetTargetHandle(target_str, target_detail_str);
959 : :
1263 960 [ + + - + ]: 176 : if (o_compression_detail && !o_compression)
1321 rhaas@postgresql.org 961 [ # # ]:UBC 0 : ereport(ERROR,
962 : : (errcode(ERRCODE_SYNTAX_ERROR),
963 : : errmsg("compression detail cannot be specified unless compression is enabled")));
964 : :
1263 rhaas@postgresql.org 965 [ + + ]:CBC 176 : if (o_compression)
966 : : {
967 : : char *error_detail;
968 : :
1243 michael@paquier.xyz 969 : 27 : parse_compress_specification(opt->compression, compression_detail_str,
970 : : &opt->compression_specification);
971 : : error_detail =
972 : 27 : validate_compress_specification(&opt->compression_specification);
1263 rhaas@postgresql.org 973 [ + + ]: 27 : if (error_detail != NULL)
974 [ + - ]: 9 : ereport(ERROR,
975 : : errcode(ERRCODE_SYNTAX_ERROR),
976 : : errmsg("invalid compression specification: %s",
977 : : error_detail));
978 : : }
5340 magnus@hagander.net 979 : 167 : }
980 : :
981 : :
982 : : /*
983 : : * SendBaseBackup() - send a complete base backup.
984 : : *
985 : : * The function will put the system into backup mode like pg_backup_start()
986 : : * does, so that the backup is consistent even though we read directly from
987 : : * the filesystem, bypassing the buffer cache.
988 : : */
989 : : void
626 rhaas@postgresql.org 990 : 182 : SendBaseBackup(BaseBackupCmd *cmd, IncrementalBackupInfo *ib)
991 : : {
992 : : basebackup_options opt;
993 : : bbsink *sink;
1152 fujii@postgresql.org 994 : 182 : SessionBackupState status = get_backup_status();
995 : :
996 [ + + ]: 182 : if (status == SESSION_BACKUP_RUNNING)
997 [ + - ]: 1 : ereport(ERROR,
998 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
999 : : errmsg("a backup is already in progress in this session")));
1000 : :
5340 magnus@hagander.net 1001 : 181 : parse_basebackup_options(cmd->options, &opt);
1002 : :
5352 1003 : 167 : WalSndSetState(WALSNDSTATE_BACKUP);
1004 : :
5353 1005 [ + - ]: 167 : if (update_process_title)
1006 : : {
1007 : : char activitymsg[50];
1008 : :
1009 : 167 : snprintf(activitymsg, sizeof(activitymsg), "sending backup \"%s\"",
1010 : : opt.label);
2005 peter@eisentraut.org 1011 : 167 : set_ps_display(activitymsg);
1012 : : }
1013 : :
1014 : : /*
1015 : : * If we're asked to perform an incremental backup and the user has not
1016 : : * supplied a manifest, that's an ERROR.
1017 : : *
1018 : : * If we're asked to perform a full backup and the user did supply a
1019 : : * manifest, just ignore it.
1020 : : */
626 rhaas@postgresql.org 1021 [ + + ]: 167 : if (!opt.incremental)
1022 : 157 : ib = NULL;
1023 [ - + ]: 10 : else if (ib == NULL)
626 rhaas@postgresql.org 1024 [ # # ]:UBC 0 : ereport(ERROR,
1025 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1026 : : errmsg("must UPLOAD_MANIFEST before performing an incremental BASE_BACKUP")));
1027 : :
1028 : : /*
1029 : : * If the target is specifically 'client' then set up to stream the backup
1030 : : * to the client; otherwise, it's being sent someplace else and should not
1031 : : * be sent to the client. BaseBackupGetSink has the job of setting up a
1032 : : * sink to send the backup data wherever it needs to go.
1033 : : */
1271 rhaas@postgresql.org 1034 :CBC 167 : sink = bbsink_copystream_new(opt.send_to_client);
1035 [ + + ]: 167 : if (opt.target_handle != NULL)
1036 : 14 : sink = BaseBackupGetSink(opt.target_handle, sink);
1037 : :
1038 : : /* Set up network throttling, if client requested it */
1401 1039 [ + + ]: 164 : if (opt.maxrate > 0)
1040 : 1 : sink = bbsink_throttle_new(sink, opt.maxrate);
1041 : :
1042 : : /* Set up server-side compression, if client requested it */
1243 michael@paquier.xyz 1043 [ + + ]: 164 : if (opt.compression == PG_COMPRESSION_GZIP)
1263 rhaas@postgresql.org 1044 : 2 : sink = bbsink_gzip_new(sink, &opt.compression_specification);
1243 michael@paquier.xyz 1045 [ + + ]: 162 : else if (opt.compression == PG_COMPRESSION_LZ4)
1263 rhaas@postgresql.org 1046 : 3 : sink = bbsink_lz4_new(sink, &opt.compression_specification);
1243 michael@paquier.xyz 1047 [ + + ]: 159 : else if (opt.compression == PG_COMPRESSION_ZSTD)
1263 rhaas@postgresql.org 1048 : 4 : sink = bbsink_zstd_new(sink, &opt.compression_specification);
1049 : :
1050 : : /* Set up progress reporting. */
32 msawada@postgresql.o 1051 :GNC 164 : sink = bbsink_progress_new(sink, opt.progress, opt.incremental);
1052 : :
1053 : : /*
1054 : : * Perform the base backup, but make sure we clean up the bbsink even if
1055 : : * an error occurs.
1056 : : */
1401 rhaas@postgresql.org 1057 [ + + ]:CBC 164 : PG_TRY();
1058 : : {
626 1059 : 164 : perform_base_backup(&opt, sink, ib);
1060 : : }
1401 1061 : 4 : PG_FINALLY();
1062 : : {
1063 : 160 : bbsink_cleanup(sink);
1064 : : }
1065 [ + + ]: 160 : PG_END_TRY();
5329 magnus@hagander.net 1066 : 156 : }
1067 : :
1068 : : /*
1069 : : * Inject a file with given name and content in the output tar stream.
1070 : : *
1071 : : * "len" can optionally be set to an arbitrary length of data sent. If set
1072 : : * to -1, the content sent is treated as a string with strlen() as length.
1073 : : */
1074 : : static void
1401 rhaas@postgresql.org 1075 : 218 : sendFileWithContent(bbsink *sink, const char *filename, const char *content,
1076 : : int len, backup_manifest_info *manifest)
1077 : : {
1078 : : struct stat statbuf;
663 michael@paquier.xyz 1079 : 218 : int bytes_done = 0;
1080 : : pg_checksum_context checksum_ctx;
1081 : :
1739 1082 [ - + ]: 218 : if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0)
1739 michael@paquier.xyz 1083 [ # # ]:UBC 0 : elog(ERROR, "could not initialize checksum of file \"%s\"",
1084 : : filename);
1085 : :
663 michael@paquier.xyz 1086 [ + - ]:CBC 218 : if (len < 0)
1087 : 218 : len = strlen(content);
1088 : :
1089 : : /*
1090 : : * Construct a stat struct for the file we're injecting in the tar.
1091 : : */
1092 : :
1093 : : /* Windows doesn't have the concept of uid and gid */
1094 : : #ifdef WIN32
1095 : : statbuf.st_uid = 0;
1096 : : statbuf.st_gid = 0;
1097 : : #else
5332 heikki.linnakangas@i 1098 : 218 : statbuf.st_uid = geteuid();
1099 : 218 : statbuf.st_gid = getegid();
1100 : : #endif
1101 : 218 : statbuf.st_mtime = time(NULL);
2709 sfrost@snowman.net 1102 : 218 : statbuf.st_mode = pg_file_create_mode;
5332 heikki.linnakangas@i 1103 : 218 : statbuf.st_size = len;
1104 : :
1401 rhaas@postgresql.org 1105 : 218 : _tarWriteHeader(sink, filename, NULL, &statbuf, false);
1106 : :
1107 [ - + ]: 218 : if (pg_checksum_update(&checksum_ctx, (uint8 *) content, len) < 0)
1401 rhaas@postgresql.org 1108 [ # # ]:UBC 0 : elog(ERROR, "could not update checksum of file \"%s\"",
1109 : : filename);
1110 : :
1401 rhaas@postgresql.org 1111 [ + + ]:CBC 390 : while (bytes_done < len)
1112 : : {
1113 : 172 : size_t remaining = len - bytes_done;
1114 : 172 : size_t nbytes = Min(sink->bbs_buffer_length, remaining);
1115 : :
1116 : 172 : memcpy(sink->bbs_buffer, content, nbytes);
1117 : 172 : bbsink_archive_contents(sink, nbytes);
1118 : 172 : bytes_done += nbytes;
999 1119 : 172 : content += nbytes;
1120 : : }
1121 : :
1401 1122 : 218 : _tarWritePadding(sink, len);
1123 : :
684 1124 : 218 : AddFileToBackupManifest(manifest, InvalidOid, filename, len,
1962 1125 : 218 : (pg_time_t) statbuf.st_mtime, &checksum_ctx);
5332 heikki.linnakangas@i 1126 : 218 : }
1127 : :
1128 : : /*
1129 : : * Include the tablespace directory pointed to by 'path' in the output tar
1130 : : * stream. If 'sizeonly' is true, we just calculate a total length and return
1131 : : * it, without actually sending anything.
1132 : : *
1133 : : * Only used to send auxiliary tablespaces, not PGDATA.
1134 : : */
1135 : : static int64
684 rhaas@postgresql.org 1136 : 78 : sendTablespace(bbsink *sink, char *path, Oid spcoid, bool sizeonly,
1137 : : backup_manifest_info *manifest, IncrementalBackupInfo *ib)
1138 : : {
1139 : : int64 size;
1140 : : char pathbuf[MAXPGPATH];
1141 : : struct stat statbuf;
1142 : :
1143 : : /*
1144 : : * 'path' points to the tablespace location, but we only want to include
1145 : : * the version directory in it that belongs to us.
1146 : : */
4548 heikki.linnakangas@i 1147 : 78 : snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path,
1148 : : TABLESPACE_VERSION_DIRECTORY);
1149 : :
1150 : : /*
1151 : : * Store a directory entry in the tar file so we get the permissions
1152 : : * right.
1153 : : */
1154 [ - + ]: 78 : if (lstat(pathbuf, &statbuf) != 0)
1155 : : {
4548 heikki.linnakangas@i 1156 [ # # ]:UBC 0 : if (errno != ENOENT)
1157 [ # # ]: 0 : ereport(ERROR,
1158 : : (errcode_for_file_access(),
1159 : : errmsg("could not stat file or directory \"%s\": %m",
1160 : : pathbuf)));
1161 : :
1162 : : /* If the tablespace went away while scanning, it's no error. */
1163 : 0 : return 0;
1164 : : }
1165 : :
1401 rhaas@postgresql.org 1166 :CBC 78 : size = _tarWriteHeader(sink, TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf,
1167 : : sizeonly);
1168 : :
1169 : : /* Send all the files in the tablespace version directory */
1170 : 78 : size += sendDir(sink, pathbuf, strlen(path), sizeonly, NIL, true, manifest,
1171 : : spcoid, ib);
1172 : :
4548 heikki.linnakangas@i 1173 : 78 : return size;
1174 : : }
1175 : :
1176 : : /*
1177 : : * Include all files from the given directory in the output tar stream. If
1178 : : * 'sizeonly' is true, we just calculate a total length and return it, without
1179 : : * actually sending anything.
1180 : : *
1181 : : * Omit any directory in the tablespaces list, to avoid backing up
1182 : : * tablespaces twice when they were created inside PGDATA.
1183 : : *
1184 : : * If sendtblspclinks is true, we need to include symlink
1185 : : * information in the tar file. If not, we can skip that
1186 : : * as it will be sent separately in the tablespace_map file.
1187 : : */
1188 : : static int64
1401 rhaas@postgresql.org 1189 : 5642 : sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
1190 : : List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest,
1191 : : Oid spcoid, IncrementalBackupInfo *ib)
1192 : : {
1193 : : DIR *dir;
1194 : : struct dirent *de;
1195 : : char pathbuf[MAXPGPATH * 2];
1196 : : struct stat statbuf;
5353 magnus@hagander.net 1197 : 5642 : int64 size = 0;
1198 : : const char *lastDir; /* Split last dir from parent path. */
662 rhaas@postgresql.org 1199 : 5642 : bool isRelationDir = false; /* Does directory contain relations? */
626 1200 : 5642 : bool isGlobalDir = false;
662 1201 : 5642 : Oid dboid = InvalidOid;
626 1202 : 5642 : BlockNumber *relative_block_numbers = NULL;
1203 : :
1204 : : /*
1205 : : * Since this array is relatively large, avoid putting it on the stack.
1206 : : * But we don't need it at all if this is not an incremental backup.
1207 : : */
1208 [ + + ]: 5642 : if (ib != NULL)
1209 : 174 : relative_block_numbers = palloc(sizeof(BlockNumber) * RELSEG_SIZE);
1210 : :
1211 : : /*
1212 : : * Determine if the current path is a database directory that can contain
1213 : : * relations.
1214 : : *
1215 : : * Start by finding the location of the delimiter between the parent path
1216 : : * and the current path.
1217 : : */
2724 teodor@sigaev.ru 1218 : 5642 : lastDir = last_dir_separator(path);
1219 : :
1220 : : /* Does this path look like a database path (i.e. all digits)? */
1221 [ + + ]: 5642 : if (lastDir != NULL &&
1222 [ + + ]: 5314 : strspn(lastDir + 1, "0123456789") == strlen(lastDir + 1))
1223 : 1050 : {
1224 : : /* Part of path that contains the parent directory. */
2690 tgl@sss.pgh.pa.us 1225 : 1050 : int parentPathLen = lastDir - path;
1226 : :
1227 : : /*
1228 : : * Mark path as a database directory if the parent path is either
1229 : : * $PGDATA/base or a tablespace version path.
1230 : : */
2724 teodor@sigaev.ru 1231 [ + + ]: 1050 : if (strncmp(path, "./base", parentPathLen) == 0 ||
1232 [ + - ]: 56 : (parentPathLen >= (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) &&
1233 [ + - ]: 56 : strncmp(lastDir - (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1),
1234 : : TABLESPACE_VERSION_DIRECTORY,
1235 : : sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) == 0))
1236 : : {
662 rhaas@postgresql.org 1237 : 1050 : isRelationDir = true;
1238 : 1050 : dboid = atooid(lastDir + 1);
1239 : : }
1240 : : }
1241 [ + + ]: 4592 : else if (strcmp(path, "./global") == 0)
1242 : : {
1243 : 328 : isRelationDir = true;
626 1244 : 328 : isGlobalDir = true;
1245 : : }
1246 : :
5353 magnus@hagander.net 1247 : 5642 : dir = AllocateDir(path);
1248 [ + + ]: 347557 : while ((de = ReadDir(dir, path)) != NULL)
1249 : : {
1250 : : int excludeIdx;
1251 : : bool excludeFound;
662 rhaas@postgresql.org 1252 : 341925 : RelFileNumber relfilenumber = InvalidRelFileNumber;
1253 : 341925 : ForkNumber relForkNum = InvalidForkNumber;
1254 : 341925 : unsigned segno = 0;
1255 : 341925 : bool isRelationFile = false;
1256 : :
1257 : : /* Skip special stuff */
5353 magnus@hagander.net 1258 [ + + + + ]: 341925 : if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
1259 : 15958 : continue;
1260 : :
1261 : : /* Skip temporary files */
heikki.linnakangas@i 1262 [ + + ]: 330656 : if (strncmp(de->d_name,
1263 : : PG_TEMP_FILE_PREFIX,
1264 : : strlen(PG_TEMP_FILE_PREFIX)) == 0)
1265 : 324 : continue;
1266 : :
1267 : : /* Skip macOS system files */
571 dgustafsson@postgres 1268 [ + + ]: 330332 : if (strcmp(de->d_name, ".DS_Store") == 0)
1269 : 65 : continue;
1270 : :
1271 : : /*
1272 : : * Check if the postmaster has signaled us to exit, and abort with an
1273 : : * error in that case. The error handler further up will call
1274 : : * do_pg_abort_backup() for us. Also check that if the backup was
1275 : : * started while still in recovery, the server wasn't promoted.
1276 : : * do_pg_backup_stop() will check that too, but it's better to stop
1277 : : * the backup early than continue to the end and fail there.
1278 : : */
4650 heikki.linnakangas@i 1279 [ + + ]: 330267 : CHECK_FOR_INTERRUPTS();
1280 [ - + ]: 330263 : if (RecoveryInProgress() != backup_started_in_recovery)
5349 magnus@hagander.net 1281 [ # # ]:UBC 0 : ereport(ERROR,
1282 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1283 : : errmsg("the standby was promoted during online backup"),
1284 : : errhint("This means that the backup being taken is corrupt "
1285 : : "and should not be used. "
1286 : : "Try taking another online backup.")));
1287 : :
1288 : : /* Scan for files that should be excluded */
3265 peter_e@gmx.net 1289 :CBC 330263 : excludeFound = false;
2021 michael@paquier.xyz 1290 [ + + ]: 2967448 : for (excludeIdx = 0; excludeFiles[excludeIdx].name != NULL; excludeIdx++)
1291 : : {
1292 : 2638468 : int cmplen = strlen(excludeFiles[excludeIdx].name);
1293 : :
1294 [ + + ]: 2638468 : if (!excludeFiles[excludeIdx].match_prefix)
1295 : 2308337 : cmplen++;
1296 [ + + ]: 2638468 : if (strncmp(de->d_name, excludeFiles[excludeIdx].name, cmplen) == 0)
1297 : : {
3265 peter_e@gmx.net 1298 [ + + ]: 1283 : elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name);
1299 : 1283 : excludeFound = true;
1300 : 1283 : break;
1301 : : }
1302 : : }
1303 : :
1304 [ + + ]: 330263 : if (excludeFound)
5353 magnus@hagander.net 1305 : 1283 : continue;
1306 : :
1307 : : /*
1308 : : * If there could be non-temporary relation files in this directory,
1309 : : * try to parse the filename.
1310 : : */
662 rhaas@postgresql.org 1311 [ + + ]: 328980 : if (isRelationDir)
1312 : : isRelationFile =
1313 : 318003 : parse_filename_for_nontemp_relation(de->d_name,
1314 : : &relfilenumber,
1315 : : &relForkNum, &segno);
1316 : :
1317 : : /* Exclude all forks for unlogged tables except the init fork */
1318 [ + + + + ]: 328980 : if (isRelationFile && relForkNum != INIT_FORKNUM)
1319 : : {
1320 : : char initForkFile[MAXPGPATH];
1321 : :
1322 : : /*
1323 : : * If any other type of fork, check if there is an init fork with
1324 : : * the same RelFileNumber. If so, the file can be excluded.
1325 : : */
1326 : 315257 : snprintf(initForkFile, sizeof(initForkFile), "%s/%u_init",
1327 : : path, relfilenumber);
1328 : :
1329 [ + + ]: 315257 : if (lstat(initForkFile, &statbuf) == 0)
1330 : : {
1331 [ - + ]: 69 : elog(DEBUG2,
1332 : : "unlogged relation file \"%s\" excluded from backup",
1333 : : de->d_name);
1334 : :
1335 : 69 : continue;
1336 : : }
1337 : : }
1338 : :
1339 : : /* Exclude temporary relations */
1340 [ + + + + ]: 328911 : if (OidIsValid(dboid) && looks_like_temp_rel_name(de->d_name))
1341 : : {
2720 teodor@sigaev.ru 1342 [ - + ]: 36 : elog(DEBUG2,
1343 : : "temporary relation file \"%s\" excluded from backup",
1344 : : de->d_name);
1345 : :
1346 : 36 : continue;
1347 : : }
1348 : :
3070 peter_e@gmx.net 1349 : 328875 : snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name);
1350 : :
1351 : : /* Skip pg_control here to back up it last */
152 fujii@postgresql.org 1352 [ + + ]: 328875 : if (strcmp(pathbuf, "./" XLOG_CONTROL_FILE) == 0)
4973 simon@2ndQuadrant.co 1353 : 326 : continue;
1354 : :
5353 magnus@hagander.net 1355 [ - + ]: 328549 : if (lstat(pathbuf, &statbuf) != 0)
1356 : : {
5353 magnus@hagander.net 1357 [ # # ]:UBC 0 : if (errno != ENOENT)
1358 [ # # ]: 0 : ereport(ERROR,
1359 : : (errcode_for_file_access(),
1360 : : errmsg("could not stat file or directory \"%s\": %m",
1361 : : pathbuf)));
1362 : :
1363 : : /* If the file went away while scanning, it's not an error. */
1364 : 0 : continue;
1365 : : }
1366 : :
1367 : : /* Scan for directories whose contents should be excluded */
3265 peter_e@gmx.net 1368 :CBC 328549 : excludeFound = false;
1369 [ + + ]: 2619345 : for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)
1370 : : {
1371 [ + + ]: 2293059 : if (strcmp(de->d_name, excludeDirContents[excludeIdx]) == 0)
1372 : : {
1373 [ + + ]: 2263 : elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name);
1425 rhaas@postgresql.org 1374 : 2263 : convert_link_to_directory(pathbuf, &statbuf);
1401 1375 : 2263 : size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,
1376 : : &statbuf, sizeonly);
3265 peter_e@gmx.net 1377 : 2263 : excludeFound = true;
1378 : 2263 : break;
1379 : : }
1380 : : }
1381 : :
1382 [ + + ]: 328549 : if (excludeFound)
1383 : 2263 : continue;
1384 : :
1385 : : /*
1386 : : * We can skip pg_wal, the WAL segments need to be fetched from the
1387 : : * WAL archive anyway. But include it as an empty directory anyway, so
1388 : : * we get permissions right.
1389 : : */
3243 rhaas@postgresql.org 1390 [ + + ]: 326286 : if (strcmp(pathbuf, "./pg_wal") == 0)
1391 : : {
1392 : : /* If pg_wal is a symlink, write it as a directory anyway */
1425 1393 : 323 : convert_link_to_directory(pathbuf, &statbuf);
1401 1394 : 323 : size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,
1395 : : &statbuf, sizeonly);
1396 : :
1397 : : /*
1398 : : * Also send archive_status and summaries directories (by
1399 : : * hackishly reusing statbuf from above ...).
1400 : : */
1401 : 323 : size += _tarWriteHeader(sink, "./pg_wal/archive_status", NULL,
1402 : : &statbuf, sizeonly);
626 1403 : 323 : size += _tarWriteHeader(sink, "./pg_wal/summaries", NULL,
1404 : : &statbuf, sizeonly);
1405 : :
3243 1406 : 323 : continue; /* don't recurse into pg_wal */
1407 : : }
1408 : :
1409 : : /* Allow symbolic links in pg_tblspc only */
1127 tmunro@postgresql.or 1410 [ + + + + ]: 325963 : if (strcmp(path, "./pg_tblspc") == 0 && S_ISLNK(statbuf.st_mode))
5353 magnus@hagander.net 1411 : 41 : {
1412 : : char linkpath[MAXPGPATH];
1413 : : int rllen;
1414 : :
5022 tgl@sss.pgh.pa.us 1415 : 41 : rllen = readlink(pathbuf, linkpath, sizeof(linkpath));
1416 [ - + ]: 41 : if (rllen < 0)
5353 magnus@hagander.net 1417 [ # # ]:UBC 0 : ereport(ERROR,
1418 : : (errcode_for_file_access(),
1419 : : errmsg("could not read symbolic link \"%s\": %m",
1420 : : pathbuf)));
5022 tgl@sss.pgh.pa.us 1421 [ - + ]:CBC 41 : if (rllen >= sizeof(linkpath))
5022 tgl@sss.pgh.pa.us 1422 [ # # ]:UBC 0 : ereport(ERROR,
1423 : : (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1424 : : errmsg("symbolic link \"%s\" target is too long",
1425 : : pathbuf)));
5022 tgl@sss.pgh.pa.us 1426 :CBC 41 : linkpath[rllen] = '\0';
1427 : :
1401 rhaas@postgresql.org 1428 : 41 : size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, linkpath,
1429 : : &statbuf, sizeonly);
1430 : : }
5353 magnus@hagander.net 1431 [ + + ]: 325922 : else if (S_ISDIR(statbuf.st_mode))
1432 : : {
4260 1433 : 5295 : bool skip_this_dir = false;
1434 : : ListCell *lc;
1435 : :
1436 : : /*
1437 : : * Store a directory entry in the tar file so we can get the
1438 : : * permissions right.
1439 : : */
1401 rhaas@postgresql.org 1440 : 5295 : size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL, &statbuf,
1441 : : sizeonly);
1442 : :
1443 : : /*
1444 : : * Call ourselves recursively for a directory, unless it happens
1445 : : * to be a separate tablespace located within PGDATA.
1446 : : */
4260 magnus@hagander.net 1447 [ + + + + : 11783 : foreach(lc, tablespaces)
+ + ]
1448 : : {
1449 : 6516 : tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
1450 : :
1451 : : /*
1452 : : * ti->rpath is the tablespace relative path within PGDATA, or
1453 : : * NULL if the tablespace has been properly located somewhere
1454 : : * else.
1455 : : *
1456 : : * Skip past the leading "./" in pathbuf when comparing.
1457 : : */
1458 [ + + + + ]: 6516 : if (ti->rpath && strcmp(ti->rpath, pathbuf + 2) == 0)
1459 : : {
1460 : 28 : skip_this_dir = true;
1461 : 28 : break;
1462 : : }
1463 : : }
1464 : :
1465 : : /*
1466 : : * skip sending directories inside pg_tblspc, if not required.
1467 : : */
3770 andrew@dunslane.net 1468 [ + + + + ]: 5295 : if (strcmp(pathbuf, "./pg_tblspc") == 0 && !sendtblspclinks)
1469 : 31 : skip_this_dir = true;
1470 : :
4260 magnus@hagander.net 1471 [ + + ]: 5295 : if (!skip_this_dir)
1401 rhaas@postgresql.org 1472 : 5236 : size += sendDir(sink, pathbuf, basepathlen, sizeonly, tablespaces,
1473 : : sendtblspclinks, manifest, spcoid, ib);
1474 : : }
5353 magnus@hagander.net 1475 [ + - ]: 320627 : else if (S_ISREG(statbuf.st_mode))
1476 : : {
4483 bruce@momjian.us 1477 : 320627 : bool sent = false;
626 rhaas@postgresql.org 1478 : 320627 : unsigned num_blocks_required = 0;
1479 : 320627 : unsigned truncation_block_length = 0;
1480 : : char tarfilenamebuf[MAXPGPATH * 2];
1481 : 320627 : char *tarfilename = pathbuf + basepathlen + 1;
1482 : 320627 : FileBackupMethod method = BACK_UP_FILE_FULLY;
1483 : :
1484 [ + + + + ]: 320627 : if (ib != NULL && isRelationFile)
1485 : : {
1486 : : Oid relspcoid;
1487 : : char *lookup_path;
1488 : :
1489 [ + + ]: 10142 : if (OidIsValid(spcoid))
1490 : : {
1491 : 9 : relspcoid = spcoid;
368 michael@paquier.xyz 1492 : 9 : lookup_path = psprintf("%s/%u/%s", PG_TBLSPC_DIR, spcoid,
1493 : : tarfilename);
1494 : : }
1495 : : else
1496 : : {
626 rhaas@postgresql.org 1497 [ + + ]: 10133 : if (isGlobalDir)
1498 : 560 : relspcoid = GLOBALTABLESPACE_OID;
1499 : : else
1500 : 9573 : relspcoid = DEFAULTTABLESPACE_OID;
1501 : 10133 : lookup_path = pstrdup(tarfilename);
1502 : : }
1503 : :
1504 : 10142 : method = GetFileBackupMethod(ib, lookup_path, dboid, relspcoid,
1505 : : relfilenumber, relForkNum,
1506 : 10142 : segno, statbuf.st_size,
1507 : : &num_blocks_required,
1508 : : relative_block_numbers,
1509 : : &truncation_block_length);
1510 [ + + ]: 10142 : if (method == BACK_UP_FILE_INCREMENTALLY)
1511 : : {
1512 : 6661 : statbuf.st_size =
1513 : 6661 : GetIncrementalFileSize(num_blocks_required);
1514 : 6661 : snprintf(tarfilenamebuf, sizeof(tarfilenamebuf),
1515 : : "%s/INCREMENTAL.%s",
1516 : 6661 : path + basepathlen + 1,
1517 : 6661 : de->d_name);
1518 : 6661 : tarfilename = tarfilenamebuf;
1519 : : }
1520 : :
1521 : 10142 : pfree(lookup_path);
1522 : : }
1523 : :
5353 magnus@hagander.net 1524 [ + + ]: 320627 : if (!sizeonly)
626 rhaas@postgresql.org 1525 [ + + ]: 157969 : sent = sendFile(sink, pathbuf, tarfilename, &statbuf,
1526 : : true, dboid, spcoid,
1527 : : relfilenumber, segno, manifest,
1528 : : num_blocks_required,
1529 : : method == BACK_UP_FILE_INCREMENTALLY ? relative_block_numbers : NULL,
1530 : : truncation_block_length);
1531 : :
4642 heikki.linnakangas@i 1532 [ + + + - ]: 320626 : if (sent || sizeonly)
1533 : : {
1534 : : /* Add size. */
1961 rhaas@postgresql.org 1535 : 320626 : size += statbuf.st_size;
1536 : :
1537 : : /* Pad to a multiple of the tar block size. */
1538 : 320626 : size += tarPaddingBytesRequired(statbuf.st_size);
1539 : :
1540 : : /* Size of the header for the file. */
1541 : 320626 : size += TAR_BLOCK_SIZE;
1542 : : }
1543 : : }
1544 : : else
5353 magnus@hagander.net 1545 [ # # ]:UBC 0 : ereport(WARNING,
1546 : : (errmsg("skipping special file \"%s\"", pathbuf)));
1547 : : }
1548 : :
626 rhaas@postgresql.org 1549 [ + + ]:CBC 5632 : if (relative_block_numbers != NULL)
1550 : 174 : pfree(relative_block_numbers);
1551 : :
5353 magnus@hagander.net 1552 : 5632 : FreeDir(dir);
1553 : 5632 : return size;
1554 : : }
1555 : :
1556 : : /*
1557 : : * Given the member, write the TAR header & send the file.
1558 : : *
1559 : : * If 'missing_ok' is true, will not throw an error if the file is not found.
1560 : : *
1561 : : * If dboid is anything other than InvalidOid then any checksum failures
1562 : : * detected will get reported to the cumulative stats system.
1563 : : *
1564 : : * If the file is to be sent incrementally, then num_incremental_blocks
1565 : : * should be the number of blocks to be sent, and incremental_blocks
1566 : : * an array of block numbers relative to the start of the current segment.
1567 : : * If the whole file is to be sent, then incremental_blocks should be NULL,
1568 : : * and num_incremental_blocks can have any value, as it will be ignored.
1569 : : *
1570 : : * Returns true if the file was successfully sent, false if 'missing_ok',
1571 : : * and the file did not exist.
1572 : : */
1573 : : static bool
1401 rhaas@postgresql.org 1574 : 158128 : sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
1575 : : struct stat *statbuf, bool missing_ok, Oid dboid, Oid spcoid,
1576 : : RelFileNumber relfilenumber, unsigned segno,
1577 : : backup_manifest_info *manifest, unsigned num_incremental_blocks,
1578 : : BlockNumber *incremental_blocks, unsigned truncation_block_length)
1579 : : {
1580 : : int fd;
2713 magnus@hagander.net 1581 : 158128 : BlockNumber blkno = 0;
1582 : 158128 : int checksum_failures = 0;
1583 : : off_t cnt;
704 rhaas@postgresql.org 1584 : 158128 : pgoff_t bytes_done = 0;
2713 magnus@hagander.net 1585 : 158128 : bool verify_checksum = false;
1586 : : pg_checksum_context checksum_ctx;
626 rhaas@postgresql.org 1587 : 158128 : int ibindex = 0;
1588 : :
1739 michael@paquier.xyz 1589 [ - + ]: 158128 : if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0)
1739 michael@paquier.xyz 1590 [ # # ]:UBC 0 : elog(ERROR, "could not initialize checksum of file \"%s\"",
1591 : : readfilename);
1592 : :
1907 rhaas@postgresql.org 1593 :CBC 158128 : fd = OpenTransientFile(readfilename, O_RDONLY | PG_BINARY);
1594 [ - + ]: 158128 : if (fd < 0)
1595 : : {
4642 heikki.linnakangas@i 1596 [ # # # # ]:UBC 0 : if (errno == ENOENT && missing_ok)
1597 : 0 : return false;
5353 magnus@hagander.net 1598 [ # # ]: 0 : ereport(ERROR,
1599 : : (errcode_for_file_access(),
1600 : : errmsg("could not open file \"%s\": %m", readfilename)));
1601 : : }
1602 : :
1401 rhaas@postgresql.org 1603 :CBC 158128 : _tarWriteHeader(sink, tarfilename, NULL, statbuf, false);
1604 : :
1605 : : /*
1606 : : * Checksums are verified in multiples of BLCKSZ, so the buffer length
1607 : : * should be a multiple of the block size as well.
1608 : : */
704 1609 [ - + ]: 158127 : Assert((sink->bbs_buffer_length % BLCKSZ) == 0);
1610 : :
1611 : : /*
1612 : : * If we weren't told not to verify checksums, and if checksums are
1613 : : * enabled for this cluster, and if this is a relation file, then verify
1614 : : * the checksum.
1615 : : */
662 1616 [ + + + - : 158127 : if (!noverify_checksums && DataChecksumsEnabled() &&
+ + ]
1617 : : RelFileNumberIsValid(relfilenumber))
1618 : 154371 : verify_checksum = true;
1619 : :
1620 : : /*
1621 : : * If we're sending an incremental file, write the file header.
1622 : : */
626 1623 [ + + ]: 158127 : if (incremental_blocks != NULL)
1624 : : {
1625 : 6661 : unsigned magic = INCREMENTAL_MAGIC;
1626 : 6661 : size_t header_bytes_done = 0;
1627 : : char padding[BLCKSZ];
1628 : : size_t paddinglen;
1629 : :
1630 : : /* Emit header data. */
1631 : 6661 : push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1632 : : &magic, sizeof(magic));
1633 : 6661 : push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1634 : : &num_incremental_blocks, sizeof(num_incremental_blocks));
1635 : 6661 : push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1636 : : &truncation_block_length, sizeof(truncation_block_length));
1637 : 6661 : push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1638 : : incremental_blocks,
1639 : : sizeof(BlockNumber) * num_incremental_blocks);
1640 : :
1641 : : /*
1642 : : * Add padding to align header to a multiple of BLCKSZ, but only if
1643 : : * the incremental file has some blocks, and the alignment is actually
1644 : : * needed (i.e. header is not already a multiple of BLCKSZ). If there
1645 : : * are no blocks we don't want to make the file unnecessarily large,
1646 : : * as that might make some filesystem optimizations impossible.
1647 : : */
510 tomas.vondra@postgre 1648 [ + + + - ]: 6661 : if ((num_incremental_blocks > 0) && (header_bytes_done % BLCKSZ != 0))
1649 : : {
519 1650 : 25 : paddinglen = (BLCKSZ - (header_bytes_done % BLCKSZ));
1651 : :
1652 : 25 : memset(padding, 0, paddinglen);
1653 : 25 : bytes_done += paddinglen;
1654 : :
1655 : 25 : push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1656 : : padding, paddinglen);
1657 : : }
1658 : :
1659 : : /* Flush out any data still in the buffer so it's again empty. */
626 rhaas@postgresql.org 1660 [ + - ]: 6661 : if (header_bytes_done > 0)
1661 : : {
1662 : 6661 : bbsink_archive_contents(sink, header_bytes_done);
1663 [ - + ]: 6661 : if (pg_checksum_update(&checksum_ctx,
1664 : 6661 : (uint8 *) sink->bbs_buffer,
1665 : : header_bytes_done) < 0)
626 rhaas@postgresql.org 1666 [ # # ]:UBC 0 : elog(ERROR, "could not update checksum of base backup");
1667 : : }
1668 : :
1669 : : /* Update our notion of file position. */
626 rhaas@postgresql.org 1670 :CBC 6661 : bytes_done += sizeof(magic);
1671 : 6661 : bytes_done += sizeof(num_incremental_blocks);
1672 : 6661 : bytes_done += sizeof(truncation_block_length);
1673 : 6661 : bytes_done += sizeof(BlockNumber) * num_incremental_blocks;
1674 : : }
1675 : :
1676 : : /*
1677 : : * Loop until we read the amount of data the caller told us to expect. The
1678 : : * file could be longer, if it was extended while we were sending it, but
1679 : : * for a base backup we can ignore such extended data. It will be restored
1680 : : * from WAL.
1681 : : */
1682 : : while (1)
1683 : : {
1684 : : /*
1685 : : * Determine whether we've read all the data that we need, and if not,
1686 : : * read some more.
1687 : : */
1688 [ + + ]: 343177 : if (incremental_blocks == NULL)
1689 : : {
1690 : 336473 : size_t remaining = statbuf->st_size - bytes_done;
1691 : :
1692 : : /*
1693 : : * If we've read the required number of bytes, then it's time to
1694 : : * stop.
1695 : : */
1696 [ + + ]: 336473 : if (bytes_done >= statbuf->st_size)
1697 : 151466 : break;
1698 : :
1699 : : /*
1700 : : * Read as many bytes as will fit in the buffer, or however many
1701 : : * are left to read, whichever is less.
1702 : : */
1703 : 185007 : cnt = read_file_data_into_buffer(sink, readfilename, fd,
1704 : : bytes_done, remaining,
1705 : 185007 : blkno + segno * RELSEG_SIZE,
1706 : : verify_checksum,
1707 : : &checksum_failures);
1708 : : }
1709 : : else
1710 : : {
1711 : : BlockNumber relative_blkno;
1712 : :
1713 : : /*
1714 : : * If we've read all the blocks, then it's time to stop.
1715 : : */
1716 [ + + ]: 6704 : if (ibindex >= num_incremental_blocks)
1717 : 6661 : break;
1718 : :
1719 : : /*
1720 : : * Read just one block, whichever one is the next that we're
1721 : : * supposed to include.
1722 : : */
1723 : 43 : relative_blkno = incremental_blocks[ibindex++];
1724 : 43 : cnt = read_file_data_into_buffer(sink, readfilename, fd,
1725 : 43 : relative_blkno * BLCKSZ,
1726 : : BLCKSZ,
1727 : 43 : relative_blkno + segno * RELSEG_SIZE,
1728 : : verify_checksum,
1729 : : &checksum_failures);
1730 : :
1731 : : /*
1732 : : * If we get a partial read, that must mean that the relation is
1733 : : * being truncated. Ultimately, it should be truncated to a
1734 : : * multiple of BLCKSZ, since this path should only be reached for
1735 : : * relation files, but we might transiently observe an
1736 : : * intermediate value.
1737 : : *
1738 : : * It should be fine to treat this just as if the entire block had
1739 : : * been truncated away - i.e. fill this and all later blocks with
1740 : : * zeroes. WAL replay will fix things up.
1741 : : */
1742 [ - + ]: 43 : if (cnt < BLCKSZ)
626 rhaas@postgresql.org 1743 :UBC 0 : break;
1744 : : }
1745 : :
1746 : : /*
1747 : : * If the amount of data we were able to read was not a multiple of
1748 : : * BLCKSZ, we cannot verify checksums, which are block-level.
1749 : : */
2713 magnus@hagander.net 1750 [ + + - + ]:CBC 185050 : if (verify_checksum && (cnt % BLCKSZ != 0))
1751 : : {
2713 magnus@hagander.net 1752 [ # # ]:UBC 0 : ereport(WARNING,
1753 : : (errmsg("could not verify checksum in file \"%s\", block "
1754 : : "%u: read buffer size %d and page size %d "
1755 : : "differ",
1756 : : readfilename, blkno, (int) cnt, BLCKSZ)));
1757 : 0 : verify_checksum = false;
1758 : : }
1759 : :
1760 : : /*
1761 : : * If we hit end-of-file, a concurrent truncation must have occurred.
1762 : : * That's not an error condition, because WAL replay will fix things
1763 : : * up.
1764 : : */
947 rhaas@postgresql.org 1765 [ - + ]:CBC 185050 : if (cnt == 0)
947 rhaas@postgresql.org 1766 :UBC 0 : break;
1767 : :
1768 : : /* Update block number and # of bytes done for next loop iteration. */
704 rhaas@postgresql.org 1769 :CBC 185050 : blkno += cnt / BLCKSZ;
1770 : 185050 : bytes_done += cnt;
1771 : :
1772 : : /*
1773 : : * Make sure incremental files with block data are properly aligned
1774 : : * (header is a multiple of BLCKSZ, blocks are BLCKSZ too).
1775 : : */
519 tomas.vondra@postgre 1776 [ + + + - : 185050 : Assert(!((incremental_blocks != NULL && num_incremental_blocks > 0) &&
- + ]
1777 : : (bytes_done % BLCKSZ != 0)));
1778 : :
1779 : : /* Archive the data we just read. */
1401 rhaas@postgresql.org 1780 : 185050 : bbsink_archive_contents(sink, cnt);
1781 : :
1782 : : /* Also feed it to the checksum machinery. */
1783 [ - + ]: 185050 : if (pg_checksum_update(&checksum_ctx,
1784 : 185050 : (uint8 *) sink->bbs_buffer, cnt) < 0)
1739 michael@paquier.xyz 1785 [ # # ]:UBC 0 : elog(ERROR, "could not update checksum of base backup");
1786 : : }
1787 : :
1788 : : /* If the file was truncated while we were sending it, pad it with zeros */
704 rhaas@postgresql.org 1789 [ - + ]:CBC 158127 : while (bytes_done < statbuf->st_size)
1790 : : {
704 rhaas@postgresql.org 1791 :UBC 0 : size_t remaining = statbuf->st_size - bytes_done;
1401 1792 : 0 : size_t nbytes = Min(sink->bbs_buffer_length, remaining);
1793 : :
1794 [ # # # # : 0 : MemSet(sink->bbs_buffer, 0, nbytes);
# # # # #
# ]
1795 [ # # ]: 0 : if (pg_checksum_update(&checksum_ctx,
1796 : 0 : (uint8 *) sink->bbs_buffer,
1797 : : nbytes) < 0)
1798 [ # # ]: 0 : elog(ERROR, "could not update checksum of base backup");
1799 : 0 : bbsink_archive_contents(sink, nbytes);
704 1800 : 0 : bytes_done += nbytes;
1801 : : }
1802 : :
1803 : : /*
1804 : : * Pad to a block boundary, per tar format requirements. (This small piece
1805 : : * of data is probably not worth throttling, and is not checksummed
1806 : : * because it's not actually part of the file.)
1807 : : */
704 rhaas@postgresql.org 1808 :CBC 158127 : _tarWritePadding(sink, bytes_done);
1809 : :
1907 1810 : 158127 : CloseTransientFile(fd);
1811 : :
2713 magnus@hagander.net 1812 [ + + ]: 158127 : if (checksum_failures > 1)
1813 : : {
1814 [ + - ]: 2 : ereport(WARNING,
1815 : : (errmsg_plural("file \"%s\" has a total of %d checksum verification failure",
1816 : : "file \"%s\" has a total of %d checksum verification failures",
1817 : : checksum_failures,
1818 : : readfilename, checksum_failures)));
1819 : :
160 andres@anarazel.de 1820 : 2 : pgstat_prepare_report_checksum_failure(dboid);
2339 magnus@hagander.net 1821 : 2 : pgstat_report_checksum_failures_in_db(dboid, checksum_failures);
1822 : : }
1823 : :
2713 1824 : 158127 : total_checksum_failures += checksum_failures;
1825 : :
1962 rhaas@postgresql.org 1826 : 158127 : AddFileToBackupManifest(manifest, spcoid, tarfilename, statbuf->st_size,
1827 : 158127 : (pg_time_t) statbuf->st_mtime, &checksum_ctx);
1828 : :
4642 heikki.linnakangas@i 1829 : 158127 : return true;
1830 : : }
1831 : :
1832 : : /*
1833 : : * Read some more data from the file into the bbsink's buffer, verifying
1834 : : * checksums as required.
1835 : : *
1836 : : * 'offset' is the file offset from which we should begin to read, and
1837 : : * 'length' is the amount of data that should be read. The actual amount
1838 : : * of data read will be less than the requested amount if the bbsink's
1839 : : * buffer isn't big enough to hold it all, or if the underlying file has
1840 : : * been truncated. The return value is the number of bytes actually read.
1841 : : *
1842 : : * 'blkno' is the block number of the first page in the bbsink's buffer
1843 : : * relative to the start of the relation.
1844 : : *
1845 : : * 'verify_checksum' indicates whether we should try to verify checksums
1846 : : * for the blocks we read. If we do this, we'll update *checksum_failures
1847 : : * and issue warnings as appropriate.
1848 : : */
1849 : : static off_t
704 rhaas@postgresql.org 1850 : 185050 : read_file_data_into_buffer(bbsink *sink, const char *readfilename, int fd,
1851 : : off_t offset, size_t length, BlockNumber blkno,
1852 : : bool verify_checksum, int *checksum_failures)
1853 : : {
1854 : : off_t cnt;
1855 : : int i;
1856 : : char *page;
1857 : :
1858 : : /* Try to read some more data. */
1859 : 185050 : cnt = basebackup_read_file(fd, sink->bbs_buffer,
1860 : 185050 : Min(sink->bbs_buffer_length, length),
1861 : : offset, readfilename, true);
1862 : :
1863 : : /* Can't verify checksums if read length is not a multiple of BLCKSZ. */
1864 [ + + - + ]: 185050 : if (!verify_checksum || (cnt % BLCKSZ) != 0)
1865 : 4371 : return cnt;
1866 : :
1867 : : /* Verify checksum for each block. */
1868 [ + + ]: 633624 : for (i = 0; i < cnt / BLCKSZ; i++)
1869 : : {
1870 : : int reread_cnt;
1871 : : uint16 expected_checksum;
1872 : :
1873 : 452945 : page = sink->bbs_buffer + BLCKSZ * i;
1874 : :
1875 : : /* If the page is OK, go on to the next one. */
1876 [ + + ]: 452945 : if (verify_page_checksum(page, sink->bbs_state->startptr, blkno + i,
1877 : : &expected_checksum))
1878 : 452931 : continue;
1879 : :
1880 : : /*
1881 : : * Retry the block on the first failure. It's possible that we read
1882 : : * the first 4K page of the block just before postgres updated the
1883 : : * entire block so it ends up looking torn to us. If, before we retry
1884 : : * the read, the concurrent write of the block finishes, the page LSN
1885 : : * will be updated and we'll realize that we should ignore this block.
1886 : : *
1887 : : * There's no guarantee that this will actually happen, though: the
1888 : : * torn write could take an arbitrarily long time to complete.
1889 : : * Retrying multiple times wouldn't fix this problem, either, though
1890 : : * it would reduce the chances of it happening in practice. The only
1891 : : * real fix here seems to be to have some kind of interlock that
1892 : : * allows us to wait until we can be certain that no write to the
1893 : : * block is in progress. Since we don't have any such thing right now,
1894 : : * we just do this and hope for the best.
1895 : : */
1896 : 14 : reread_cnt =
1897 : 14 : basebackup_read_file(fd, sink->bbs_buffer + BLCKSZ * i,
1898 : 14 : BLCKSZ, offset + BLCKSZ * i,
1899 : : readfilename, false);
1900 [ - + ]: 14 : if (reread_cnt == 0)
1901 : : {
1902 : : /*
1903 : : * If we hit end-of-file, a concurrent truncation must have
1904 : : * occurred, so reduce cnt to reflect only the blocks already
1905 : : * processed and break out of this loop.
1906 : : */
704 rhaas@postgresql.org 1907 :UBC 0 : cnt = BLCKSZ * i;
1908 : 0 : break;
1909 : : }
1910 : :
1911 : : /* If the page now looks OK, go on to the next one. */
704 rhaas@postgresql.org 1912 [ - + ]:CBC 14 : if (verify_page_checksum(page, sink->bbs_state->startptr, blkno + i,
1913 : : &expected_checksum))
704 rhaas@postgresql.org 1914 :UBC 0 : continue;
1915 : :
1916 : : /* Handle checksum failure. */
704 rhaas@postgresql.org 1917 :CBC 14 : (*checksum_failures)++;
1918 [ + + ]: 14 : if (*checksum_failures <= 5)
1919 [ + - ]: 12 : ereport(WARNING,
1920 : : (errmsg("checksum verification failed in "
1921 : : "file \"%s\", block %u: calculated "
1922 : : "%X but expected %X",
1923 : : readfilename, blkno + i, expected_checksum,
1924 : : ((PageHeader) page)->pd_checksum)));
1925 [ + + ]: 14 : if (*checksum_failures == 5)
1926 [ + - ]: 2 : ereport(WARNING,
1927 : : (errmsg("further checksum verification "
1928 : : "failures in file \"%s\" will not "
1929 : : "be reported", readfilename)));
1930 : : }
1931 : :
1932 : 180679 : return cnt;
1933 : : }
1934 : :
1935 : : /*
1936 : : * Push data into a bbsink.
1937 : : *
1938 : : * It's better, when possible, to read data directly into the bbsink's buffer,
1939 : : * rather than using this function to copy it into the buffer; this function is
1940 : : * for cases where that approach is not practical.
1941 : : *
1942 : : * bytes_done should point to a count of the number of bytes that are
1943 : : * currently used in the bbsink's buffer. Upon return, the bytes identified by
1944 : : * data and length will have been copied into the bbsink's buffer, flushing
1945 : : * as required, and *bytes_done will have been updated accordingly. If the
1946 : : * buffer was flushed, the previous contents will also have been fed to
1947 : : * checksum_ctx.
1948 : : *
1949 : : * Note that after one or more calls to this function it is the caller's
1950 : : * responsibility to perform any required final flush.
1951 : : */
1952 : : static void
626 1953 : 26669 : push_to_sink(bbsink *sink, pg_checksum_context *checksum_ctx,
1954 : : size_t *bytes_done, void *data, size_t length)
1955 : : {
1956 [ + + ]: 26669 : while (length > 0)
1957 : : {
1958 : : size_t bytes_to_copy;
1959 : :
1960 : : /*
1961 : : * We use < here rather than <= so that if the data exactly fills the
1962 : : * remaining buffer space, we trigger a flush now.
1963 : : */
1964 [ + - ]: 20033 : if (length < sink->bbs_buffer_length - *bytes_done)
1965 : : {
1966 : : /* Append remaining data to buffer. */
1967 : 20033 : memcpy(sink->bbs_buffer + *bytes_done, data, length);
1968 : 20033 : *bytes_done += length;
1969 : 20033 : return;
1970 : : }
1971 : :
1972 : : /* Copy until buffer is full and flush it. */
626 rhaas@postgresql.org 1973 :UBC 0 : bytes_to_copy = sink->bbs_buffer_length - *bytes_done;
1974 : 0 : memcpy(sink->bbs_buffer + *bytes_done, data, bytes_to_copy);
1975 : 0 : data = ((char *) data) + bytes_to_copy;
1976 : 0 : length -= bytes_to_copy;
1977 : 0 : bbsink_archive_contents(sink, sink->bbs_buffer_length);
1978 [ # # ]: 0 : if (pg_checksum_update(checksum_ctx, (uint8 *) sink->bbs_buffer,
1979 : : sink->bbs_buffer_length) < 0)
1980 [ # # ]: 0 : elog(ERROR, "could not update checksum");
1981 : 0 : *bytes_done = 0;
1982 : : }
1983 : : }
1984 : :
1985 : : /*
1986 : : * Try to verify the checksum for the provided page, if it seems appropriate
1987 : : * to do so.
1988 : : *
1989 : : * Returns true if verification succeeds or if we decide not to check it,
1990 : : * and false if verification fails. When return false, it also sets
1991 : : * *expected_checksum to the computed value.
1992 : : */
1993 : : static bool
704 rhaas@postgresql.org 1994 :CBC 452959 : verify_page_checksum(Page page, XLogRecPtr start_lsn, BlockNumber blkno,
1995 : : uint16 *expected_checksum)
1996 : : {
1997 : : PageHeader phdr;
1998 : : uint16 checksum;
1999 : :
2000 : : /*
2001 : : * Only check pages which have not been modified since the start of the
2002 : : * base backup. Otherwise, they might have been written only halfway and
2003 : : * the checksum would not be valid. However, replaying WAL would
2004 : : * reinstate the correct page in this case. We also skip completely new
2005 : : * pages, since they don't have a checksum yet.
2006 : : */
2007 [ + + - + ]: 452959 : if (PageIsNew(page) || PageGetLSN(page) >= start_lsn)
2008 : 81 : return true;
2009 : :
2010 : : /* Perform the actual checksum calculation. */
2011 : 452878 : checksum = pg_checksum_page(page, blkno);
2012 : :
2013 : : /* See whether it matches the value from the page. */
2014 : 452878 : phdr = (PageHeader) page;
2015 [ + + ]: 452878 : if (phdr->pd_checksum == checksum)
2016 : 452850 : return true;
2017 : 28 : *expected_checksum = checksum;
2018 : 28 : return false;
2019 : : }
2020 : :
2021 : : static int64
1401 2022 : 167014 : _tarWriteHeader(bbsink *sink, const char *filename, const char *linktarget,
2023 : : struct stat *statbuf, bool sizeonly)
2024 : : {
2025 : : enum tarError rc;
2026 : :
3265 peter_e@gmx.net 2027 [ + + ]: 167014 : if (!sizeonly)
2028 : : {
2029 : : /*
2030 : : * As of this writing, the smallest supported block size is 1kB, which
2031 : : * is twice TAR_BLOCK_SIZE. Since the buffer size is required to be a
2032 : : * multiple of BLCKSZ, it should be safe to assume that the buffer is
2033 : : * large enough to fit an entire tar block. We double-check by means
2034 : : * of these assertions.
2035 : : */
2036 : : StaticAssertDecl(TAR_BLOCK_SIZE <= BLCKSZ,
2037 : : "BLCKSZ too small for tar block");
1401 rhaas@postgresql.org 2038 [ - + ]: 162628 : Assert(sink->bbs_buffer_length >= TAR_BLOCK_SIZE);
2039 : :
2040 : 162628 : rc = tarCreateHeader(sink->bbs_buffer, filename, linktarget,
2041 : : statbuf->st_size, statbuf->st_mode,
2042 : : statbuf->st_uid, statbuf->st_gid,
2043 : : statbuf->st_mtime);
2044 : :
3265 peter_e@gmx.net 2045 [ + + - - ]: 162628 : switch (rc)
2046 : : {
2047 : 162627 : case TAR_OK:
2048 : 162627 : break;
2049 : 1 : case TAR_NAME_TOO_LONG:
2050 [ + - ]: 1 : ereport(ERROR,
2051 : : (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
2052 : : errmsg("file name too long for tar format: \"%s\"",
2053 : : filename)));
2054 : : break;
3265 peter_e@gmx.net 2055 :UBC 0 : case TAR_SYMLINK_TOO_LONG:
2056 [ # # ]: 0 : ereport(ERROR,
2057 : : (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
2058 : : errmsg("symbolic link target too long for tar format: "
2059 : : "file name \"%s\", target \"%s\"",
2060 : : filename, linktarget)));
2061 : : break;
2062 : 0 : default:
2063 [ # # ]: 0 : elog(ERROR, "unrecognized tar error: %d", rc);
2064 : : }
2065 : :
1401 rhaas@postgresql.org 2066 :CBC 162627 : bbsink_archive_contents(sink, TAR_BLOCK_SIZE);
2067 : : }
2068 : :
2069 : 167013 : return TAR_BLOCK_SIZE;
2070 : : }
2071 : :
2072 : : /*
2073 : : * Pad with zero bytes out to a multiple of TAR_BLOCK_SIZE.
2074 : : */
2075 : : static void
2076 : 158345 : _tarWritePadding(bbsink *sink, int len)
2077 : : {
2078 : 158345 : int pad = tarPaddingBytesRequired(len);
2079 : :
2080 : : /*
2081 : : * As in _tarWriteHeader, it should be safe to assume that the buffer is
2082 : : * large enough that we don't need to do this in multiple chunks.
2083 : : */
2084 [ - + ]: 158345 : Assert(sink->bbs_buffer_length >= TAR_BLOCK_SIZE);
2085 [ - + ]: 158345 : Assert(pad <= TAR_BLOCK_SIZE);
2086 : :
2087 [ + + ]: 158345 : if (pad > 0)
2088 : : {
2089 [ + - + + : 32173 : MemSet(sink->bbs_buffer, 0, pad);
+ - + - +
+ ]
2090 : 8956 : bbsink_archive_contents(sink, pad);
2091 : : }
4209 alvherre@alvh.no-ip. 2092 : 158345 : }
2093 : :
2094 : : /*
2095 : : * If the entry in statbuf is a link, then adjust statbuf to make it look like a
2096 : : * directory, so that it will be written that way.
2097 : : */
2098 : : static void
1401 rhaas@postgresql.org 2099 : 2586 : convert_link_to_directory(const char *pathbuf, struct stat *statbuf)
2100 : : {
2101 : : /* If symlink, write it as a directory anyway */
2102 [ + + ]: 2586 : if (S_ISLNK(statbuf->st_mode))
2103 : 62 : statbuf->st_mode = S_IFDIR | pg_dir_create_mode;
2013 fujii@postgresql.org 2104 : 2586 : }
2105 : :
2106 : : /*
2107 : : * Read some data from a file, setting a wait event and reporting any error
2108 : : * encountered.
2109 : : *
2110 : : * If partial_read_ok is false, also report an error if the number of bytes
2111 : : * read is not equal to the number of bytes requested.
2112 : : *
2113 : : * Returns the number of bytes read.
2114 : : */
2115 : : static ssize_t
1907 rhaas@postgresql.org 2116 : 196328 : basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset,
2117 : : const char *filename, bool partial_read_ok)
2118 : : {
2119 : : ssize_t rc;
2120 : :
2121 : 196328 : pgstat_report_wait_start(WAIT_EVENT_BASEBACKUP_READ);
1073 tmunro@postgresql.or 2122 : 196328 : rc = pg_pread(fd, buf, nbytes, offset);
1907 rhaas@postgresql.org 2123 : 196328 : pgstat_report_wait_end();
2124 : :
2125 [ - + ]: 196328 : if (rc < 0)
1907 rhaas@postgresql.org 2126 [ # # ]:UBC 0 : ereport(ERROR,
2127 : : (errcode_for_file_access(),
2128 : : errmsg("could not read file \"%s\": %m", filename)));
1907 rhaas@postgresql.org 2129 [ + + + - :CBC 196328 : if (!partial_read_ok && rc > 0 && rc != nbytes)
- + ]
1907 rhaas@postgresql.org 2130 [ # # ]:UBC 0 : ereport(ERROR,
2131 : : (errcode_for_file_access(),
2132 : : errmsg("could not read file \"%s\": read %zd of %zu",
2133 : : filename, rc, nbytes)));
2134 : :
1907 rhaas@postgresql.org 2135 :CBC 196328 : return rc;
2136 : : }
|
