Age Owner Branch data TLA Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * basebackup.c
4 : : * code for taking a base backup and streaming it to a standby
5 : : *
6 : : * Portions Copyright (c) 2010-2026, PostgreSQL Global Development Group
7 : : *
8 : : * IDENTIFICATION
9 : : * src/backend/backup/basebackup.c
10 : : *
11 : : *-------------------------------------------------------------------------
12 : : */
13 : : #include "postgres.h"
14 : :
15 : : #include <sys/stat.h>
16 : : #include <unistd.h>
17 : : #include <time.h>
18 : :
19 : : #include "access/xlog_internal.h"
20 : : #include "access/xlogbackup.h"
21 : : #include "backup/backup_manifest.h"
22 : : #include "backup/basebackup.h"
23 : : #include "backup/basebackup_incremental.h"
24 : : #include "backup/basebackup_sink.h"
25 : : #include "backup/basebackup_target.h"
26 : : #include "catalog/pg_tablespace_d.h"
27 : : #include "commands/defrem.h"
28 : : #include "common/compression.h"
29 : : #include "common/file_perm.h"
30 : : #include "common/file_utils.h"
31 : : #include "lib/stringinfo.h"
32 : : #include "miscadmin.h"
33 : : #include "nodes/pg_list.h"
34 : : #include "pgstat.h"
35 : : #include "pgtar.h"
36 : : #include "postmaster/syslogger.h"
37 : : #include "postmaster/walsummarizer.h"
38 : : #include "replication/slot.h"
39 : : #include "replication/walsender.h"
40 : : #include "replication/walsender_private.h"
41 : : #include "storage/bufpage.h"
42 : : #include "storage/checksum.h"
43 : : #include "storage/dsm_impl.h"
44 : : #include "storage/ipc.h"
45 : : #include "storage/reinit.h"
46 : : #include "utils/builtins.h"
47 : : #include "utils/guc.h"
48 : : #include "utils/ps_status.h"
49 : : #include "utils/relcache.h"
50 : : #include "utils/resowner.h"
51 : : #include "utils/wait_event.h"
52 : :
53 : : /*
54 : : * How much data do we want to send in one CopyData message? Note that
55 : : * this may also result in reading the underlying files in chunks of this
56 : : * size.
57 : : *
58 : : * NB: The buffer size is required to be a multiple of the system block
59 : : * size, so use that value instead if it's bigger than our preference.
60 : : */
61 : : #define SINK_BUFFER_LENGTH Max(32768, BLCKSZ)
62 : :
63 : : typedef struct
64 : : {
65 : : const char *label;
66 : : bool progress;
67 : : bool fastcheckpoint;
68 : : bool nowait;
69 : : bool includewal;
70 : : bool incremental;
71 : : uint32 maxrate;
72 : : bool sendtblspcmapfile;
73 : : bool send_to_client;
74 : : bool use_copytblspc;
75 : : BaseBackupTargetHandle *target_handle;
76 : : backup_manifest_option manifest;
77 : : pg_compress_algorithm compression;
78 : : pg_compress_specification compression_specification;
79 : : pg_checksum_type manifest_checksum_type;
80 : : } basebackup_options;
81 : :
82 : : #define TAR_NUM_TERMINATION_BLOCKS 2
83 : :
84 : : StaticAssertDecl(TAR_NUM_TERMINATION_BLOCKS * TAR_BLOCK_SIZE <= BLCKSZ,
85 : : "BLCKSZ too small for " CppAsString2(TAR_NUM_TERMINATION_BLOCKS) " tar termination blocks");
86 : :
87 : : static int64 sendTablespace(bbsink *sink, char *path, Oid spcoid, bool sizeonly,
88 : : struct backup_manifest_info *manifest,
89 : : IncrementalBackupInfo *ib);
90 : : static int64 sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
91 : : List *tablespaces, bool sendtblspclinks,
92 : : backup_manifest_info *manifest, Oid spcoid,
93 : : IncrementalBackupInfo *ib);
94 : : static bool sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
95 : : struct stat *statbuf, bool missing_ok,
96 : : Oid dboid, Oid spcoid, RelFileNumber relfilenumber,
97 : : unsigned segno,
98 : : backup_manifest_info *manifest,
99 : : unsigned num_incremental_blocks,
100 : : BlockNumber *incremental_blocks,
101 : : unsigned truncation_block_length);
102 : : static off_t read_file_data_into_buffer(bbsink *sink,
103 : : const char *readfilename, int fd,
104 : : off_t offset, size_t length,
105 : : BlockNumber blkno,
106 : : bool verify_checksum,
107 : : int *checksum_failures);
108 : : static void push_to_sink(bbsink *sink, pg_checksum_context *checksum_ctx,
109 : : size_t *bytes_done, void *data, size_t length);
110 : : static bool verify_page_checksum(Page page, XLogRecPtr start_lsn,
111 : : BlockNumber blkno,
112 : : uint16 *expected_checksum);
113 : : static void sendFileWithContent(bbsink *sink, const char *filename,
114 : : const char *content, int len,
115 : : backup_manifest_info *manifest);
116 : : static int64 _tarWriteHeader(bbsink *sink, const char *filename,
117 : : const char *linktarget, struct stat *statbuf,
118 : : bool sizeonly);
119 : : static void _tarWritePadding(bbsink *sink, int len);
120 : : static void convert_link_to_directory(const char *pathbuf, struct stat *statbuf);
121 : : static void perform_base_backup(basebackup_options *opt, bbsink *sink,
122 : : IncrementalBackupInfo *ib);
123 : : static void parse_basebackup_options(List *options, basebackup_options *opt);
124 : : static int compareWalFileNames(const ListCell *a, const ListCell *b);
125 : : static ssize_t basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset,
126 : : const char *filename, bool partial_read_ok);
127 : :
128 : : /* Was the backup currently in-progress initiated in recovery mode? */
129 : : static bool backup_started_in_recovery = false;
130 : :
131 : : /* Total number of checksum failures during base backup. */
132 : : static long long int total_checksum_failures;
133 : :
134 : : /* Do not verify checksums. */
135 : : static bool noverify_checksums = false;
136 : :
137 : : /*
138 : : * Definition of one element part of an exclusion list, used for paths part
139 : : * of checksum validation or base backups. "name" is the name of the file
140 : : * or path to check for exclusion. If "match_prefix" is true, any items
141 : : * matching the name as prefix are excluded.
142 : : */
143 : : struct exclude_list_item
144 : : {
145 : : const char *name;
146 : : bool match_prefix;
147 : : };
148 : :
149 : : /*
150 : : * The contents of these directories are removed or recreated during server
151 : : * start so they are not included in backups. The directories themselves are
152 : : * kept and included as empty to preserve access permissions.
153 : : *
154 : : * Note: this list should be kept in sync with the filter lists in pg_rewind's
155 : : * filemap.c.
156 : : */
157 : : static const char *const excludeDirContents[] =
158 : : {
159 : : /*
160 : : * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped
161 : : * because extensions like pg_stat_statements store data there.
162 : : */
163 : : PG_STAT_TMP_DIR,
164 : :
165 : : /*
166 : : * It is generally not useful to backup the contents of this directory
167 : : * even if the intention is to restore to another primary. See backup.sgml
168 : : * for a more detailed description.
169 : : */
170 : : PG_REPLSLOT_DIR,
171 : :
172 : : /* Contents removed on startup, see dsm_cleanup_for_mmap(). */
173 : : PG_DYNSHMEM_DIR,
174 : :
175 : : /* Contents removed on startup, see AsyncShmemInit(). */
176 : : "pg_notify",
177 : :
178 : : /*
179 : : * Old contents are loaded for possible debugging but are not required for
180 : : * normal operation, see SerialInit().
181 : : */
182 : : "pg_serial",
183 : :
184 : : /* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */
185 : : "pg_snapshots",
186 : :
187 : : /* Contents zeroed on startup, see StartupSUBTRANS(). */
188 : : "pg_subtrans",
189 : :
190 : : /* end of list */
191 : : NULL
192 : : };
193 : :
194 : : /*
195 : : * List of files excluded from backups.
196 : : */
197 : : static const struct exclude_list_item excludeFiles[] =
198 : : {
199 : : /* Skip auto conf temporary file. */
200 : : {PG_AUTOCONF_FILENAME ".tmp", false},
201 : :
202 : : /* Skip current log file temporary file */
203 : : {LOG_METAINFO_DATAFILE_TMP, false},
204 : :
205 : : /*
206 : : * Skip relation cache because it is rebuilt on startup. This includes
207 : : * temporary files.
208 : : */
209 : : {RELCACHE_INIT_FILENAME, true},
210 : :
211 : : /*
212 : : * backup_label and tablespace_map should not exist in a running cluster
213 : : * capable of doing an online backup, but exclude them just in case.
214 : : */
215 : : {BACKUP_LABEL_FILE, false},
216 : : {TABLESPACE_MAP, false},
217 : :
218 : : /*
219 : : * If there's a backup_manifest, it belongs to a backup that was used to
220 : : * start this server. It is *not* correct for this backup. Our
221 : : * backup_manifest is injected into the backup separately if users want
222 : : * it.
223 : : */
224 : : {"backup_manifest", false},
225 : :
226 : : {"postmaster.pid", false},
227 : : {"postmaster.opts", false},
228 : :
229 : : /* end of list */
230 : : {NULL, false}
231 : : };
232 : :
233 : : /*
234 : : * Actually do a base backup for the specified tablespaces.
235 : : *
236 : : * This is split out mainly to avoid complaints about "variable might be
237 : : * clobbered by longjmp" from stupider versions of gcc.
238 : : */
239 : : static void
867 rhaas@postgresql.org 240 :CBC 183 : perform_base_backup(basebackup_options *opt, bbsink *sink,
241 : : IncrementalBackupInfo *ib)
242 : : {
243 : : bbsink_state state;
244 : : XLogRecPtr endptr;
245 : : TimeLineID endtli;
246 : : backup_manifest_info manifest;
247 : : BackupState *backup_state;
248 : : StringInfoData tablespace_map;
249 : :
250 : : /* Initial backup state, insofar as we know it now. */
1642 251 : 183 : state.tablespaces = NIL;
252 : 183 : state.tablespace_num = 0;
253 : 183 : state.bytes_done = 0;
254 : 183 : state.bytes_total = 0;
255 : 183 : state.bytes_total_is_valid = false;
256 : :
257 : : /* we're going to use a BufFile, so we need a ResourceOwner */
574 andres@anarazel.de 258 [ - + ]: 183 : Assert(AuxProcessResourceOwner != NULL);
259 [ + + - + ]: 183 : Assert(CurrentResourceOwner == AuxProcessResourceOwner ||
260 : : CurrentResourceOwner == NULL);
261 : 183 : CurrentResourceOwner = AuxProcessResourceOwner;
262 : :
4891 heikki.linnakangas@i 263 : 183 : backup_started_in_recovery = RecoveryInProgress();
264 : :
2203 rhaas@postgresql.org 265 : 183 : InitializeBackupManifest(&manifest, opt->manifest,
266 : : opt->manifest_checksum_type);
267 : :
2954 magnus@hagander.net 268 : 183 : total_checksum_failures = 0;
269 : :
270 : : /* Allocate backup related variables. */
146 michael@paquier.xyz 271 :GNC 183 : backup_state = palloc0_object(BackupState);
180 drowley@postgresql.o 272 : 183 : initStringInfo(&tablespace_map);
273 : :
1642 rhaas@postgresql.org 274 :CBC 183 : basebackup_progress_wait_checkpoint();
1317 michael@paquier.xyz 275 : 183 : do_pg_backup_start(opt->label, opt->fastcheckpoint, &state.tablespaces,
276 : : backup_state, &tablespace_map);
277 : :
278 : 183 : state.startptr = backup_state->startpoint;
279 : 183 : state.starttli = backup_state->starttli;
280 : :
281 : : /*
282 : : * Once do_pg_backup_start has been called, ensure that any failure causes
283 : : * us to abort the backup so we don't "leak" a backup counter. For this
284 : : * reason, *all* functionality between do_pg_backup_start() and the end of
285 : : * do_pg_backup_stop() should be inside the error cleanup block!
286 : : */
287 : :
2329 rhaas@postgresql.org 288 [ + + ]: 183 : PG_ENSURE_ERROR_CLEANUP(do_pg_abort_backup, BoolGetDatum(false));
289 : : {
290 : : ListCell *lc;
291 : : tablespaceinfo *newti;
292 : :
293 : : /* If this is an incremental backup, execute preparatory steps. */
867 294 [ + + ]: 183 : if (ib != NULL)
295 : 11 : PrepareForIncrementalBackup(ib, backup_state);
296 : :
297 : : /* Add a node for the base directory at the end */
146 michael@paquier.xyz 298 :GNC 183 : newti = palloc0_object(tablespaceinfo);
1308 drowley@postgresql.o 299 :CBC 183 : newti->size = -1;
300 : 183 : state.tablespaces = lappend(state.tablespaces, newti);
301 : :
302 : : /*
303 : : * Calculate the total backup size by summing up the size of each
304 : : * tablespace
305 : : */
2254 fujii@postgresql.org 306 [ + - ]: 183 : if (opt->progress)
307 : : {
1642 rhaas@postgresql.org 308 : 183 : basebackup_progress_estimate_backup_size();
309 : :
310 [ + - + + : 405 : foreach(lc, state.tablespaces)
+ + ]
311 : : {
2254 fujii@postgresql.org 312 : 222 : tablespaceinfo *tmp = (tablespaceinfo *) lfirst(lc);
313 : :
2148 rhaas@postgresql.org 314 [ + + ]: 222 : if (tmp->path == NULL)
1642 315 : 183 : tmp->size = sendDir(sink, ".", 1, true, state.tablespaces,
316 : : true, NULL, InvalidOid, NULL);
317 : : else
318 : 39 : tmp->size = sendTablespace(sink, tmp->path, tmp->oid, true,
319 : : NULL, NULL);
320 : 222 : state.bytes_total += tmp->size;
321 : : }
322 : 183 : state.bytes_total_is_valid = true;
323 : : }
324 : :
325 : : /* notify basebackup sink about start of backup */
326 : 183 : bbsink_begin_backup(sink, &state, SINK_BUFFER_LENGTH);
327 : :
328 : : /* Send off our tablespaces one by one */
329 [ + - + + : 400 : foreach(lc, state.tablespaces)
+ + ]
330 : : {
5593 tgl@sss.pgh.pa.us 331 : 222 : tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
332 : :
5214 simon@2ndQuadrant.co 333 [ + + ]: 222 : if (ti->path == NULL)
334 : : {
335 : : struct stat statbuf;
1819 tgl@sss.pgh.pa.us 336 : 183 : bool sendtblspclinks = true;
337 : : char *backup_label;
338 : :
1642 rhaas@postgresql.org 339 : 183 : bbsink_begin_archive(sink, "base.tar");
340 : :
341 : : /* In the main tar, include the backup_label first... */
1317 michael@paquier.xyz 342 : 183 : backup_label = build_backup_content(backup_state, false);
343 : 183 : sendFileWithContent(sink, BACKUP_LABEL_FILE,
344 : : backup_label, -1, &manifest);
345 : 183 : pfree(backup_label);
346 : :
347 : : /* Then the tablespace_map file, if required... */
2148 rhaas@postgresql.org 348 [ + + ]: 183 : if (opt->sendtblspcmapfile)
349 : : {
1317 michael@paquier.xyz 350 : 33 : sendFileWithContent(sink, TABLESPACE_MAP,
180 drowley@postgresql.o 351 :GNC 33 : tablespace_map.data, -1, &manifest);
2148 rhaas@postgresql.org 352 :CBC 33 : sendtblspclinks = false;
353 : : }
354 : :
355 : : /* Then the bulk of the files... */
1642 356 : 183 : sendDir(sink, ".", 1, false, state.tablespaces,
357 : : sendtblspclinks, &manifest, InvalidOid, ib);
358 : :
359 : : /* ... and pg_control after everything else. */
5214 simon@2ndQuadrant.co 360 [ - + ]: 178 : if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
5214 simon@2ndQuadrant.co 361 [ # # ]:UBC 0 : ereport(ERROR,
362 : : (errcode_for_file_access(),
363 : : errmsg("could not stat file \"%s\": %m",
364 : : XLOG_CONTROL_FILE)));
1642 rhaas@postgresql.org 365 :CBC 178 : sendFile(sink, XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf,
366 : : false, InvalidOid, InvalidOid,
367 : : InvalidRelFileNumber, 0, &manifest, 0, NULL, 0);
368 : : }
369 : : else
370 : : {
925 371 : 39 : char *archive_name = psprintf("%u.tar", ti->oid);
372 : :
1642 373 : 39 : bbsink_begin_archive(sink, archive_name);
374 : :
867 375 : 39 : sendTablespace(sink, ti->path, ti->oid, false, &manifest, ib);
376 : : }
377 : :
378 : : /*
379 : : * If we're including WAL, and this is the main data directory we
380 : : * don't treat this as the end of the tablespace. Instead, we will
381 : : * include the xlog files below and stop afterwards. This is safe
382 : : * since the main data directory is always sent *last*.
383 : : */
5574 magnus@hagander.net 384 [ + + + + ]: 217 : if (opt->includewal && ti->path == NULL)
385 : : {
1642 rhaas@postgresql.org 386 [ - + ]: 22 : Assert(lnext(state.tablespaces, lc) == NULL);
387 : : }
388 : : else
389 : : {
390 : : /* Properly terminate the tarfile. */
78 peter@eisentraut.org 391 :GNC 195 : memset(sink->bbs_buffer, 0, TAR_NUM_TERMINATION_BLOCKS * TAR_BLOCK_SIZE);
392 : 195 : bbsink_archive_contents(sink, TAR_NUM_TERMINATION_BLOCKS * TAR_BLOCK_SIZE);
393 : :
394 : : /* OK, that's the end of the archive. */
1642 rhaas@postgresql.org 395 :CBC 195 : bbsink_end_archive(sink);
396 : : }
397 : : }
398 : :
399 : 178 : basebackup_progress_wait_wal_archive(&state);
1317 michael@paquier.xyz 400 : 178 : do_pg_backup_stop(backup_state, !opt->nowait);
401 : :
402 : 178 : endptr = backup_state->stoppoint;
403 : 178 : endtli = backup_state->stoptli;
404 : :
405 : : /* Deallocate backup-related variables. */
180 drowley@postgresql.o 406 :GNC 178 : pfree(tablespace_map.data);
1317 michael@paquier.xyz 407 :CBC 178 : pfree(backup_state);
408 : : }
2329 rhaas@postgresql.org 409 [ - + ]: 179 : PG_END_ENSURE_ERROR_CLEANUP(do_pg_abort_backup, BoolGetDatum(false));
410 : :
411 : :
5574 magnus@hagander.net 412 [ + + ]: 178 : if (opt->includewal)
413 : : {
414 : : /*
415 : : * We've left the last tar file "open", so we can now append the
416 : : * required WAL files to it.
417 : : */
418 : : char pathbuf[MAXPGPATH];
419 : : XLogSegNo segno;
420 : : XLogSegNo startsegno;
421 : : XLogSegNo endsegno;
422 : : struct stat statbuf;
4870 heikki.linnakangas@i 423 : 22 : List *historyFileList = NIL;
424 : 22 : List *walFileList = NIL;
425 : : char firstoff[MAXFNAMELEN];
426 : : char lastoff[MAXFNAMELEN];
427 : : DIR *dir;
428 : : struct dirent *de;
429 : : ListCell *lc;
430 : : TimeLineID tli;
431 : :
1642 rhaas@postgresql.org 432 : 22 : basebackup_progress_transfer_wal();
433 : :
434 : : /*
435 : : * I'd rather not worry about timelines here, so scan pg_wal and
436 : : * include all WAL files in the range between 'startptr' and 'endptr',
437 : : * regardless of the timeline the file is stamped with. If there are
438 : : * some spurious WAL files belonging to timelines that don't belong in
439 : : * this server's history, they will be included too. Normally there
440 : : * shouldn't be such files, but if there are, there's little harm in
441 : : * including them.
442 : : */
443 : 22 : XLByteToSeg(state.startptr, startsegno, wal_segment_size);
444 : 22 : XLogFileName(firstoff, state.starttli, startsegno, wal_segment_size);
3150 andres@anarazel.de 445 : 22 : XLByteToPrevSeg(endptr, endsegno, wal_segment_size);
1649 rhaas@postgresql.org 446 : 22 : XLogFileName(lastoff, endtli, endsegno, wal_segment_size);
447 : :
3484 448 : 22 : dir = AllocateDir("pg_wal");
449 [ + + ]: 157 : while ((de = ReadDir(dir, "pg_wal")) != NULL)
450 : : {
451 : : /* Does it look like a WAL segment, and is it in the range? */
4015 heikki.linnakangas@i 452 [ + + ]: 135 : if (IsXLogFileName(de->d_name) &&
4870 453 [ + - ]: 47 : strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
454 [ + + ]: 47 : strcmp(de->d_name + 8, lastoff + 8) <= 0)
455 : : {
456 : 22 : walFileList = lappend(walFileList, pstrdup(de->d_name));
457 : : }
458 : : /* Does it look like a timeline history file? */
4015 459 [ - + ]: 113 : else if (IsTLHistoryFileName(de->d_name))
460 : : {
4870 heikki.linnakangas@i 461 :UBC 0 : historyFileList = lappend(historyFileList, pstrdup(de->d_name));
462 : : }
463 : : }
4870 heikki.linnakangas@i 464 :CBC 22 : FreeDir(dir);
465 : :
466 : : /*
467 : : * Before we go any further, check that none of the WAL segments we
468 : : * need were removed.
469 : : */
1642 rhaas@postgresql.org 470 : 22 : CheckXLogRemoved(startsegno, state.starttli);
471 : :
472 : : /*
473 : : * Sort the WAL filenames. We want to send the files in order from
474 : : * oldest to newest, to reduce the chance that a file is recycled
475 : : * before we get a chance to send it over.
476 : : */
2485 tgl@sss.pgh.pa.us 477 : 22 : list_sort(walFileList, compareWalFileNames);
478 : :
479 : : /*
480 : : * There must be at least one xlog file in the pg_wal directory, since
481 : : * we are doing backup-including-xlog.
482 : : */
483 [ - + ]: 22 : if (walFileList == NIL)
4637 magnus@hagander.net 484 [ # # ]:UBC 0 : ereport(ERROR,
485 : : (errmsg("could not find any WAL files")));
486 : :
487 : : /*
488 : : * Sanity check: the first and last segment should cover startptr and
489 : : * endptr, with no gaps in between.
490 : : */
2485 tgl@sss.pgh.pa.us 491 :CBC 22 : XLogFromFileName((char *) linitial(walFileList),
492 : : &tli, &segno, wal_segment_size);
4870 heikki.linnakangas@i 493 [ - + ]: 22 : if (segno != startsegno)
494 : : {
495 : : char startfname[MAXFNAMELEN];
496 : :
1642 rhaas@postgresql.org 497 :UBC 0 : XLogFileName(startfname, state.starttli, startsegno,
498 : : wal_segment_size);
4870 heikki.linnakangas@i 499 [ # # ]: 0 : ereport(ERROR,
500 : : (errmsg("could not find WAL file \"%s\"", startfname)));
501 : : }
2485 tgl@sss.pgh.pa.us 502 [ + - + + :CBC 44 : foreach(lc, walFileList)
+ + ]
503 : : {
504 : 22 : char *walFileName = (char *) lfirst(lc);
4724 bruce@momjian.us 505 : 22 : XLogSegNo currsegno = segno;
506 : 22 : XLogSegNo nextsegno = segno + 1;
507 : :
2485 tgl@sss.pgh.pa.us 508 : 22 : XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
4870 heikki.linnakangas@i 509 [ + - - + ]: 22 : if (!(nextsegno == segno || currsegno == segno))
510 : : {
511 : : char nextfname[MAXFNAMELEN];
512 : :
1649 rhaas@postgresql.org 513 :UBC 0 : XLogFileName(nextfname, tli, nextsegno, wal_segment_size);
4870 heikki.linnakangas@i 514 [ # # ]: 0 : ereport(ERROR,
515 : : (errmsg("could not find WAL file \"%s\"", nextfname)));
516 : : }
517 : : }
4870 heikki.linnakangas@i 518 [ - + ]:CBC 22 : if (segno != endsegno)
519 : : {
520 : : char endfname[MAXFNAMELEN];
521 : :
1649 rhaas@postgresql.org 522 :UBC 0 : XLogFileName(endfname, endtli, endsegno, wal_segment_size);
4870 heikki.linnakangas@i 523 [ # # ]: 0 : ereport(ERROR,
524 : : (errmsg("could not find WAL file \"%s\"", endfname)));
525 : : }
526 : :
527 : : /* Ok, we have everything we need. Send the WAL files. */
2485 tgl@sss.pgh.pa.us 528 [ + - + + :CBC 44 : foreach(lc, walFileList)
+ + ]
529 : : {
530 : 22 : char *walFileName = (char *) lfirst(lc);
531 : : int fd;
532 : : ssize_t cnt;
4870 heikki.linnakangas@i 533 : 22 : pgoff_t len = 0;
534 : :
2485 tgl@sss.pgh.pa.us 535 : 22 : snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFileName);
536 : 22 : XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
537 : :
2148 rhaas@postgresql.org 538 : 22 : fd = OpenTransientFile(pathbuf, O_RDONLY | PG_BINARY);
539 [ - + ]: 22 : if (fd < 0)
540 : : {
2871 michael@paquier.xyz 541 :UBC 0 : int save_errno = errno;
542 : :
543 : : /*
544 : : * Most likely reason for this is that the file was already
545 : : * removed by a checkpoint, so check for that to get a better
546 : : * error message.
547 : : */
4870 heikki.linnakangas@i 548 : 0 : CheckXLogRemoved(segno, tli);
549 : :
2871 michael@paquier.xyz 550 : 0 : errno = save_errno;
4870 heikki.linnakangas@i 551 [ # # ]: 0 : ereport(ERROR,
552 : : (errcode_for_file_access(),
553 : : errmsg("could not open file \"%s\": %m", pathbuf)));
554 : : }
555 : :
2148 rhaas@postgresql.org 556 [ - + ]:CBC 22 : if (fstat(fd, &statbuf) != 0)
4870 heikki.linnakangas@i 557 [ # # ]:UBC 0 : ereport(ERROR,
558 : : (errcode_for_file_access(),
559 : : errmsg("could not stat file \"%s\": %m",
560 : : pathbuf)));
3150 andres@anarazel.de 561 [ - + ]:CBC 22 : if (statbuf.st_size != wal_segment_size)
562 : : {
4870 heikki.linnakangas@i 563 :UBC 0 : CheckXLogRemoved(segno, tli);
564 [ # # ]: 0 : ereport(ERROR,
565 : : (errcode_for_file_access(),
566 : : errmsg("unexpected WAL file size \"%s\"", walFileName)));
567 : : }
568 : :
569 : : /* send the WAL file itself */
1642 rhaas@postgresql.org 570 :CBC 22 : _tarWriteHeader(sink, pathbuf, NULL, &statbuf, false);
571 : :
572 : 22 : while ((cnt = basebackup_read_file(fd, sink->bbs_buffer,
573 : 11264 : Min(sink->bbs_buffer_length,
574 : : wal_segment_size - len),
2148 575 [ + - ]: 11264 : len, pathbuf, true)) > 0)
576 : : {
4870 heikki.linnakangas@i 577 : 11264 : CheckXLogRemoved(segno, tli);
1642 rhaas@postgresql.org 578 : 11264 : bbsink_archive_contents(sink, cnt);
579 : :
4870 heikki.linnakangas@i 580 : 11264 : len += cnt;
581 : :
3150 andres@anarazel.de 582 [ + + ]: 11264 : if (len == wal_segment_size)
4870 heikki.linnakangas@i 583 : 22 : break;
584 : : }
585 : :
3150 andres@anarazel.de 586 [ - + ]: 22 : if (len != wal_segment_size)
587 : : {
4870 heikki.linnakangas@i 588 :UBC 0 : CheckXLogRemoved(segno, tli);
589 [ # # ]: 0 : ereport(ERROR,
590 : : (errcode_for_file_access(),
591 : : errmsg("unexpected WAL file size \"%s\"", walFileName)));
592 : : }
593 : :
594 : : /*
595 : : * wal_segment_size is a multiple of TAR_BLOCK_SIZE, so no need
596 : : * for padding.
597 : : */
2202 rhaas@postgresql.org 598 [ - + ]:CBC 22 : Assert(wal_segment_size % TAR_BLOCK_SIZE == 0);
599 : :
2148 600 : 22 : CloseTransientFile(fd);
601 : :
602 : : /*
603 : : * Mark file as archived, otherwise files can get archived again
604 : : * after promotion of a new node. This is in line with
605 : : * walreceiver.c always doing an XLogArchiveForceDone() after a
606 : : * complete segment.
607 : : */
2485 tgl@sss.pgh.pa.us 608 : 22 : StatusFilePath(pathbuf, walFileName, ".done");
904 michael@paquier.xyz 609 : 22 : sendFileWithContent(sink, pathbuf, "", -1, &manifest);
610 : : }
611 : :
612 : : /*
613 : : * Send timeline history files too. Only the latest timeline history
614 : : * file is required for recovery, and even that only if there happens
615 : : * to be a timeline switch in the first WAL segment that contains the
616 : : * checkpoint record, or if we're taking a base backup from a standby
617 : : * server and the target timeline changes while the backup is taken.
618 : : * But they are small and highly useful for debugging purposes, so
619 : : * better include them all, always.
620 : : */
4870 heikki.linnakangas@i 621 [ - + - - : 22 : foreach(lc, historyFileList)
- + ]
622 : : {
4724 bruce@momjian.us 623 :UBC 0 : char *fname = lfirst(lc);
624 : :
4870 heikki.linnakangas@i 625 : 0 : snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
626 : :
627 [ # # ]: 0 : if (lstat(pathbuf, &statbuf) != 0)
628 [ # # ]: 0 : ereport(ERROR,
629 : : (errcode_for_file_access(),
630 : : errmsg("could not stat file \"%s\": %m", pathbuf)));
631 : :
925 rhaas@postgresql.org 632 : 0 : sendFile(sink, pathbuf, pathbuf, &statbuf, false,
633 : : InvalidOid, InvalidOid, InvalidRelFileNumber, 0,
634 : : &manifest, 0, NULL, 0);
635 : :
636 : : /* unconditionally mark file as archived */
4140 andres@anarazel.de 637 : 0 : StatusFilePath(pathbuf, fname, ".done");
904 michael@paquier.xyz 638 : 0 : sendFileWithContent(sink, pathbuf, "", -1, &manifest);
639 : : }
640 : :
641 : : /* Properly terminate the tar file. */
78 peter@eisentraut.org 642 :GNC 22 : memset(sink->bbs_buffer, 0, TAR_NUM_TERMINATION_BLOCKS * TAR_BLOCK_SIZE);
643 : 22 : bbsink_archive_contents(sink, TAR_NUM_TERMINATION_BLOCKS * TAR_BLOCK_SIZE);
644 : :
645 : : /* OK, that's the end of the archive. */
1642 rhaas@postgresql.org 646 :CBC 22 : bbsink_end_archive(sink);
647 : : }
648 : :
649 : 178 : AddWALInfoToBackupManifest(&manifest, state.startptr, state.starttli,
650 : : endptr, endtli);
651 : :
652 : 178 : SendBackupManifest(&manifest, sink);
653 : :
654 : 178 : bbsink_end_backup(sink, endptr, endtli);
655 : :
2954 magnus@hagander.net 656 [ + + ]: 178 : if (total_checksum_failures)
657 : : {
658 [ + + ]: 3 : if (total_checksum_failures > 1)
659 [ + - ]: 2 : ereport(WARNING,
660 : : (errmsg_plural("%lld total checksum verification failure",
661 : : "%lld total checksum verification failures",
662 : : total_checksum_failures,
663 : : total_checksum_failures)));
664 : :
665 [ + - ]: 3 : ereport(ERROR,
666 : : (errcode(ERRCODE_DATA_CORRUPTED),
667 : : errmsg("checksum verification failure during base backup")));
668 : : }
669 : :
670 : : /*
671 : : * Make sure to free the manifest before the resource owners as manifests
672 : : * use cryptohash contexts that may depend on resource owners (like
673 : : * OpenSSL).
674 : : */
1978 michael@paquier.xyz 675 : 175 : FreeBackupManifest(&manifest);
676 : :
677 : : /* clean up the resource owner we created */
574 andres@anarazel.de 678 : 175 : ReleaseAuxProcessResources(true);
679 : :
1642 rhaas@postgresql.org 680 : 175 : basebackup_progress_done();
5593 tgl@sss.pgh.pa.us 681 : 175 : }
682 : :
683 : : /*
684 : : * list_sort comparison function, to compare log/seg portion of WAL segment
685 : : * filenames, ignoring the timeline portion.
686 : : */
687 : : static int
2485 tgl@sss.pgh.pa.us 688 :UBC 0 : compareWalFileNames(const ListCell *a, const ListCell *b)
689 : : {
690 : 0 : char *fna = (char *) lfirst(a);
691 : 0 : char *fnb = (char *) lfirst(b);
692 : :
4870 heikki.linnakangas@i 693 : 0 : return strcmp(fna + 8, fnb + 8);
694 : : }
695 : :
696 : : /*
697 : : * Parse the base backup options passed down by the parser
698 : : */
699 : : static void
5581 magnus@hagander.net 700 :CBC 200 : parse_basebackup_options(List *options, basebackup_options *opt)
701 : : {
702 : : ListCell *lopt;
703 : 200 : bool o_label = false;
704 : 200 : bool o_progress = false;
1673 rhaas@postgresql.org 705 : 200 : bool o_checkpoint = false;
5564 magnus@hagander.net 706 : 200 : bool o_nowait = false;
5574 707 : 200 : bool o_wal = false;
867 rhaas@postgresql.org 708 : 200 : bool o_incremental = false;
4450 alvherre@alvh.no-ip. 709 : 200 : bool o_maxrate = false;
4011 andrew@dunslane.net 710 : 200 : bool o_tablespace_map = false;
2954 magnus@hagander.net 711 : 200 : bool o_noverify_checksums = false;
2223 rhaas@postgresql.org 712 : 200 : bool o_manifest = false;
713 : 200 : bool o_manifest_checksums = false;
1568 714 : 200 : bool o_target = false;
1631 715 : 200 : bool o_target_detail = false;
1512 716 : 200 : char *target_str = NULL;
717 : 200 : char *target_detail_str = NULL;
1562 718 : 200 : bool o_compression = false;
1504 719 : 200 : bool o_compression_detail = false;
720 : 200 : char *compression_detail_str = NULL;
721 : :
5579 magnus@hagander.net 722 [ + - + - : 2200 : MemSet(opt, 0, sizeof(*opt));
+ - + - +
+ ]
2223 rhaas@postgresql.org 723 : 200 : opt->manifest = MANIFEST_OPTION_NO;
724 : 200 : opt->manifest_checksum_type = CHECKSUM_TYPE_CRC32C;
1484 michael@paquier.xyz 725 : 200 : opt->compression = PG_COMPRESSION_NONE;
726 : 200 : opt->compression_specification.algorithm = PG_COMPRESSION_NONE;
727 : :
5581 magnus@hagander.net 728 [ + - + + : 1522 : foreach(lopt, options)
+ + ]
729 : : {
730 : 1325 : DefElem *defel = (DefElem *) lfirst(lopt);
731 : :
732 [ + + ]: 1325 : if (strcmp(defel->defname, "label") == 0)
733 : : {
734 [ - + ]: 200 : if (o_label)
5581 magnus@hagander.net 735 [ # # ]:UBC 0 : ereport(ERROR,
736 : : (errcode(ERRCODE_SYNTAX_ERROR),
737 : : errmsg("duplicate option \"%s\"", defel->defname)));
1673 rhaas@postgresql.org 738 :CBC 200 : opt->label = defGetString(defel);
5581 magnus@hagander.net 739 : 200 : o_label = true;
740 : : }
741 [ + + ]: 1125 : else if (strcmp(defel->defname, "progress") == 0)
742 : : {
743 [ - + ]: 200 : if (o_progress)
5581 magnus@hagander.net 744 [ # # ]:UBC 0 : ereport(ERROR,
745 : : (errcode(ERRCODE_SYNTAX_ERROR),
746 : : errmsg("duplicate option \"%s\"", defel->defname)));
1673 rhaas@postgresql.org 747 :CBC 200 : opt->progress = defGetBoolean(defel);
5581 magnus@hagander.net 748 : 200 : o_progress = true;
749 : : }
1673 rhaas@postgresql.org 750 [ + + ]: 925 : else if (strcmp(defel->defname, "checkpoint") == 0)
751 : : {
752 : 190 : char *optval = defGetString(defel);
753 : :
754 [ - + ]: 190 : if (o_checkpoint)
5581 magnus@hagander.net 755 [ # # ]:UBC 0 : ereport(ERROR,
756 : : (errcode(ERRCODE_SYNTAX_ERROR),
757 : : errmsg("duplicate option \"%s\"", defel->defname)));
1673 rhaas@postgresql.org 758 [ + - ]:CBC 190 : if (pg_strcasecmp(optval, "fast") == 0)
759 : 190 : opt->fastcheckpoint = true;
1673 rhaas@postgresql.org 760 [ # # ]:UBC 0 : else if (pg_strcasecmp(optval, "spread") == 0)
761 : 0 : opt->fastcheckpoint = false;
762 : : else
763 [ # # ]: 0 : ereport(ERROR,
764 : : (errcode(ERRCODE_SYNTAX_ERROR),
765 : : errmsg("unrecognized checkpoint type: \"%s\"",
766 : : optval)));
1673 rhaas@postgresql.org 767 :CBC 190 : o_checkpoint = true;
768 : : }
769 [ + + ]: 735 : else if (strcmp(defel->defname, "wait") == 0)
770 : : {
5564 magnus@hagander.net 771 [ - + ]: 191 : if (o_nowait)
5564 magnus@hagander.net 772 [ # # ]:UBC 0 : ereport(ERROR,
773 : : (errcode(ERRCODE_SYNTAX_ERROR),
774 : : errmsg("duplicate option \"%s\"", defel->defname)));
1673 rhaas@postgresql.org 775 :CBC 191 : opt->nowait = !defGetBoolean(defel);
5564 magnus@hagander.net 776 : 191 : o_nowait = true;
777 : : }
5574 778 [ + + ]: 544 : else if (strcmp(defel->defname, "wal") == 0)
779 : : {
780 [ - + ]: 26 : if (o_wal)
5574 magnus@hagander.net 781 [ # # ]:UBC 0 : ereport(ERROR,
782 : : (errcode(ERRCODE_SYNTAX_ERROR),
783 : : errmsg("duplicate option \"%s\"", defel->defname)));
1673 rhaas@postgresql.org 784 :CBC 26 : opt->includewal = defGetBoolean(defel);
5574 magnus@hagander.net 785 : 26 : o_wal = true;
786 : : }
867 rhaas@postgresql.org 787 [ + + ]: 518 : else if (strcmp(defel->defname, "incremental") == 0)
788 : : {
789 [ - + ]: 11 : if (o_incremental)
867 rhaas@postgresql.org 790 [ # # ]:UBC 0 : ereport(ERROR,
791 : : (errcode(ERRCODE_SYNTAX_ERROR),
792 : : errmsg("duplicate option \"%s\"", defel->defname)));
867 rhaas@postgresql.org 793 :CBC 11 : opt->incremental = defGetBoolean(defel);
794 [ + - - + ]: 11 : if (opt->incremental && !summarize_wal)
867 rhaas@postgresql.org 795 [ # # ]:UBC 0 : ereport(ERROR,
796 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
797 : : errmsg("incremental backups cannot be taken unless WAL summarization is enabled")));
867 rhaas@postgresql.org 798 :CBC 11 : o_incremental = true;
799 : : }
4450 alvherre@alvh.no-ip. 800 [ + + ]: 507 : else if (strcmp(defel->defname, "max_rate") == 0)
801 : : {
802 : : int64 maxrate;
803 : :
804 [ - + ]: 1 : if (o_maxrate)
4450 alvherre@alvh.no-ip. 805 [ # # ]:UBC 0 : ereport(ERROR,
806 : : (errcode(ERRCODE_SYNTAX_ERROR),
807 : : errmsg("duplicate option \"%s\"", defel->defname)));
808 : :
1673 rhaas@postgresql.org 809 :CBC 1 : maxrate = defGetInt64(defel);
4450 alvherre@alvh.no-ip. 810 [ + - - + ]: 1 : if (maxrate < MAX_RATE_LOWER || maxrate > MAX_RATE_UPPER)
4450 alvherre@alvh.no-ip. 811 [ # # ]:UBC 0 : ereport(ERROR,
812 : : (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
813 : : errmsg("%" PRId64 " is outside the valid range for parameter \"%s\" (%d .. %d)",
814 : : maxrate, "MAX_RATE", MAX_RATE_LOWER, MAX_RATE_UPPER)));
815 : :
4450 alvherre@alvh.no-ip. 816 :CBC 1 : opt->maxrate = (uint32) maxrate;
817 : 1 : o_maxrate = true;
818 : : }
4011 andrew@dunslane.net 819 [ + + ]: 506 : else if (strcmp(defel->defname, "tablespace_map") == 0)
820 : : {
821 [ - + ]: 39 : if (o_tablespace_map)
4011 andrew@dunslane.net 822 [ # # ]:UBC 0 : ereport(ERROR,
823 : : (errcode(ERRCODE_SYNTAX_ERROR),
824 : : errmsg("duplicate option \"%s\"", defel->defname)));
1673 rhaas@postgresql.org 825 :CBC 39 : opt->sendtblspcmapfile = defGetBoolean(defel);
4011 andrew@dunslane.net 826 : 39 : o_tablespace_map = true;
827 : : }
1673 rhaas@postgresql.org 828 [ + + ]: 467 : else if (strcmp(defel->defname, "verify_checksums") == 0)
829 : : {
2954 magnus@hagander.net 830 [ - + ]: 1 : if (o_noverify_checksums)
2954 magnus@hagander.net 831 [ # # ]:UBC 0 : ereport(ERROR,
832 : : (errcode(ERRCODE_SYNTAX_ERROR),
833 : : errmsg("duplicate option \"%s\"", defel->defname)));
1673 rhaas@postgresql.org 834 :CBC 1 : noverify_checksums = !defGetBoolean(defel);
2954 magnus@hagander.net 835 : 1 : o_noverify_checksums = true;
836 : : }
2223 rhaas@postgresql.org 837 [ + + ]: 466 : else if (strcmp(defel->defname, "manifest") == 0)
838 : : {
1673 839 : 199 : char *optval = defGetString(defel);
840 : : bool manifest_bool;
841 : :
2223 842 [ - + ]: 199 : if (o_manifest)
2223 rhaas@postgresql.org 843 [ # # ]:UBC 0 : ereport(ERROR,
844 : : (errcode(ERRCODE_SYNTAX_ERROR),
845 : : errmsg("duplicate option \"%s\"", defel->defname)));
2223 rhaas@postgresql.org 846 [ + + ]:CBC 199 : if (parse_bool(optval, &manifest_bool))
847 : : {
848 [ + - ]: 198 : if (manifest_bool)
849 : 198 : opt->manifest = MANIFEST_OPTION_YES;
850 : : else
2223 rhaas@postgresql.org 851 :UBC 0 : opt->manifest = MANIFEST_OPTION_NO;
852 : : }
2223 rhaas@postgresql.org 853 [ + - ]:GBC 1 : else if (pg_strcasecmp(optval, "force-encode") == 0)
854 : 1 : opt->manifest = MANIFEST_OPTION_FORCE_ENCODE;
855 : : else
2223 rhaas@postgresql.org 856 [ # # ]:UBC 0 : ereport(ERROR,
857 : : (errcode(ERRCODE_SYNTAX_ERROR),
858 : : errmsg("unrecognized manifest option: \"%s\"",
859 : : optval)));
2223 rhaas@postgresql.org 860 :CBC 199 : o_manifest = true;
861 : : }
862 [ + + ]: 267 : else if (strcmp(defel->defname, "manifest_checksums") == 0)
863 : : {
1673 864 : 14 : char *optval = defGetString(defel);
865 : :
2223 866 [ - + ]: 14 : if (o_manifest_checksums)
2223 rhaas@postgresql.org 867 [ # # ]:UBC 0 : ereport(ERROR,
868 : : (errcode(ERRCODE_SYNTAX_ERROR),
869 : : errmsg("duplicate option \"%s\"", defel->defname)));
2223 rhaas@postgresql.org 870 [ + + ]:CBC 14 : if (!pg_checksum_parse_type(optval,
871 : : &opt->manifest_checksum_type))
872 [ + - ]: 2 : ereport(ERROR,
873 : : (errcode(ERRCODE_SYNTAX_ERROR),
874 : : errmsg("unrecognized checksum algorithm: \"%s\"",
875 : : optval)));
876 : 12 : o_manifest_checksums = true;
877 : : }
1568 878 [ + + ]: 253 : else if (strcmp(defel->defname, "target") == 0)
879 : : {
880 [ - + ]: 198 : if (o_target)
1568 rhaas@postgresql.org 881 [ # # ]:UBC 0 : ereport(ERROR,
882 : : (errcode(ERRCODE_SYNTAX_ERROR),
883 : : errmsg("duplicate option \"%s\"", defel->defname)));
1512 rhaas@postgresql.org 884 :CBC 198 : target_str = defGetString(defel);
1568 885 : 198 : o_target = true;
886 : : }
1631 887 [ + + ]: 55 : else if (strcmp(defel->defname, "target_detail") == 0)
888 : : {
889 : 10 : char *optval = defGetString(defel);
890 : :
891 [ - + ]: 10 : if (o_target_detail)
1631 rhaas@postgresql.org 892 [ # # ]:UBC 0 : ereport(ERROR,
893 : : (errcode(ERRCODE_SYNTAX_ERROR),
894 : : errmsg("duplicate option \"%s\"", defel->defname)));
1512 rhaas@postgresql.org 895 :CBC 10 : target_detail_str = optval;
1631 896 : 10 : o_target_detail = true;
897 : : }
1562 898 [ + + ]: 45 : else if (strcmp(defel->defname, "compression") == 0)
899 : : {
900 : 30 : char *optval = defGetString(defel);
901 : :
902 [ - + ]: 30 : if (o_compression)
1562 rhaas@postgresql.org 903 [ # # ]:UBC 0 : ereport(ERROR,
904 : : (errcode(ERRCODE_SYNTAX_ERROR),
905 : : errmsg("duplicate option \"%s\"", defel->defname)));
1484 michael@paquier.xyz 906 [ + + ]:CBC 30 : if (!parse_compress_algorithm(optval, &opt->compression))
1562 rhaas@postgresql.org 907 [ + - ]: 1 : ereport(ERROR,
908 : : (errcode(ERRCODE_SYNTAX_ERROR),
909 : : errmsg("unrecognized compression algorithm: \"%s\"",
910 : : optval)));
911 : 29 : o_compression = true;
912 : : }
1504 913 [ + - ]: 15 : else if (strcmp(defel->defname, "compression_detail") == 0)
914 : : {
915 [ - + ]: 15 : if (o_compression_detail)
1562 rhaas@postgresql.org 916 [ # # ]:UBC 0 : ereport(ERROR,
917 : : (errcode(ERRCODE_SYNTAX_ERROR),
918 : : errmsg("duplicate option \"%s\"", defel->defname)));
1504 rhaas@postgresql.org 919 :CBC 15 : compression_detail_str = defGetString(defel);
920 : 15 : o_compression_detail = true;
921 : : }
922 : : else
1562 rhaas@postgresql.org 923 [ # # ]:UBC 0 : ereport(ERROR,
924 : : (errcode(ERRCODE_SYNTAX_ERROR),
925 : : errmsg("unrecognized base backup option: \"%s\"",
926 : : defel->defname)));
927 : : }
928 : :
5581 magnus@hagander.net 929 [ - + ]:CBC 197 : if (opt->label == NULL)
5581 magnus@hagander.net 930 :UBC 0 : opt->label = "base backup";
2223 rhaas@postgresql.org 931 [ + + ]:CBC 197 : if (opt->manifest == MANIFEST_OPTION_NO)
932 : : {
933 [ - + ]: 1 : if (o_manifest_checksums)
2223 rhaas@postgresql.org 934 [ # # ]:UBC 0 : ereport(ERROR,
935 : : (errcode(ERRCODE_SYNTAX_ERROR),
936 : : errmsg("manifest checksums require a backup manifest")));
2223 rhaas@postgresql.org 937 :CBC 1 : opt->manifest_checksum_type = CHECKSUM_TYPE_NONE;
938 : : }
939 : :
1512 940 [ - + ]: 197 : if (target_str == NULL)
941 : : {
1512 rhaas@postgresql.org 942 [ # # ]:UBC 0 : if (target_detail_str != NULL)
1631 943 [ # # ]: 0 : ereport(ERROR,
944 : : (errcode(ERRCODE_SYNTAX_ERROR),
945 : : errmsg("target detail cannot be used without target")));
1512 946 : 0 : opt->use_copytblspc = true;
947 : 0 : opt->send_to_client = true;
948 : : }
1512 rhaas@postgresql.org 949 [ + + ]:CBC 197 : else if (strcmp(target_str, "client") == 0)
950 : : {
951 [ - + ]: 181 : if (target_detail_str != NULL)
1631 rhaas@postgresql.org 952 [ # # ]:UBC 0 : ereport(ERROR,
953 : : (errcode(ERRCODE_SYNTAX_ERROR),
954 : : errmsg("target \"%s\" does not accept a target detail",
955 : : target_str)));
1512 rhaas@postgresql.org 956 :CBC 181 : opt->send_to_client = true;
957 : : }
958 : : else
959 : 14 : opt->target_handle =
960 : 16 : BaseBackupGetTargetHandle(target_str, target_detail_str);
961 : :
1504 962 [ + + - + ]: 195 : if (o_compression_detail && !o_compression)
1562 rhaas@postgresql.org 963 [ # # ]:UBC 0 : ereport(ERROR,
964 : : (errcode(ERRCODE_SYNTAX_ERROR),
965 : : errmsg("compression detail cannot be specified unless compression is enabled")));
966 : :
1504 rhaas@postgresql.org 967 [ + + ]:CBC 195 : if (o_compression)
968 : : {
969 : : char *error_detail;
970 : :
1484 michael@paquier.xyz 971 : 27 : parse_compress_specification(opt->compression, compression_detail_str,
972 : : &opt->compression_specification);
973 : : error_detail =
974 : 27 : validate_compress_specification(&opt->compression_specification);
1504 rhaas@postgresql.org 975 [ + + ]: 27 : if (error_detail != NULL)
976 [ + - ]: 9 : ereport(ERROR,
977 : : errcode(ERRCODE_SYNTAX_ERROR),
978 : : errmsg("invalid compression specification: %s",
979 : : error_detail));
980 : : }
5581 magnus@hagander.net 981 : 186 : }
982 : :
983 : :
984 : : /*
985 : : * SendBaseBackup() - send a complete base backup.
986 : : *
987 : : * The function will put the system into backup mode like pg_backup_start()
988 : : * does, so that the backup is consistent even though we read directly from
989 : : * the filesystem, bypassing the buffer cache.
990 : : */
991 : : void
867 rhaas@postgresql.org 992 : 201 : SendBaseBackup(BaseBackupCmd *cmd, IncrementalBackupInfo *ib)
993 : : {
994 : : basebackup_options opt;
995 : : bbsink *sink;
1393 fujii@postgresql.org 996 : 201 : SessionBackupState status = get_backup_status();
997 : :
998 [ + + ]: 201 : if (status == SESSION_BACKUP_RUNNING)
999 [ + - ]: 1 : ereport(ERROR,
1000 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1001 : : errmsg("a backup is already in progress in this session")));
1002 : :
5581 magnus@hagander.net 1003 : 200 : parse_basebackup_options(cmd->options, &opt);
1004 : :
5593 1005 : 186 : WalSndSetState(WALSNDSTATE_BACKUP);
1006 : :
5594 1007 [ + - ]: 186 : if (update_process_title)
1008 : : {
1009 : : char activitymsg[50];
1010 : :
1011 : 186 : snprintf(activitymsg, sizeof(activitymsg), "sending backup \"%s\"",
1012 : : opt.label);
2246 peter@eisentraut.org 1013 : 186 : set_ps_display(activitymsg);
1014 : : }
1015 : :
1016 : : /*
1017 : : * If we're asked to perform an incremental backup and the user has not
1018 : : * supplied a manifest, that's an ERROR.
1019 : : *
1020 : : * If we're asked to perform a full backup and the user did supply a
1021 : : * manifest, just ignore it.
1022 : : */
867 rhaas@postgresql.org 1023 [ + + ]: 186 : if (!opt.incremental)
1024 : 175 : ib = NULL;
1025 [ - + ]: 11 : else if (ib == NULL)
867 rhaas@postgresql.org 1026 [ # # ]:UBC 0 : ereport(ERROR,
1027 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1028 : : errmsg("must UPLOAD_MANIFEST before performing an incremental BASE_BACKUP")));
1029 : :
1030 : : /*
1031 : : * If the target is specifically 'client' then set up to stream the backup
1032 : : * to the client; otherwise, it's being sent someplace else and should not
1033 : : * be sent to the client. BaseBackupGetSink has the job of setting up a
1034 : : * sink to send the backup data wherever it needs to go.
1035 : : */
1512 rhaas@postgresql.org 1036 :CBC 186 : sink = bbsink_copystream_new(opt.send_to_client);
1037 [ + + ]: 186 : if (opt.target_handle != NULL)
1038 : 14 : sink = BaseBackupGetSink(opt.target_handle, sink);
1039 : :
1040 : : /* Set up network throttling, if client requested it */
1642 1041 [ + + ]: 183 : if (opt.maxrate > 0)
1042 : 1 : sink = bbsink_throttle_new(sink, opt.maxrate);
1043 : :
1044 : : /* Set up server-side compression, if client requested it */
1484 michael@paquier.xyz 1045 [ + + ]: 183 : if (opt.compression == PG_COMPRESSION_GZIP)
1504 rhaas@postgresql.org 1046 : 2 : sink = bbsink_gzip_new(sink, &opt.compression_specification);
1484 michael@paquier.xyz 1047 [ + + ]: 181 : else if (opt.compression == PG_COMPRESSION_LZ4)
1504 rhaas@postgresql.org 1048 : 3 : sink = bbsink_lz4_new(sink, &opt.compression_specification);
1484 michael@paquier.xyz 1049 [ + + ]: 178 : else if (opt.compression == PG_COMPRESSION_ZSTD)
1504 rhaas@postgresql.org 1050 : 4 : sink = bbsink_zstd_new(sink, &opt.compression_specification);
1051 : :
1052 : : /* Set up progress reporting. */
273 msawada@postgresql.o 1053 :GNC 183 : sink = bbsink_progress_new(sink, opt.progress, opt.incremental);
1054 : :
1055 : : /*
1056 : : * Perform the base backup, but make sure we clean up the bbsink even if
1057 : : * an error occurs.
1058 : : */
1642 rhaas@postgresql.org 1059 [ + + ]:CBC 183 : PG_TRY();
1060 : : {
867 1061 : 183 : perform_base_backup(&opt, sink, ib);
1062 : : }
1642 1063 : 4 : PG_FINALLY();
1064 : : {
1065 : 179 : bbsink_cleanup(sink);
1066 : : }
1067 [ + + ]: 179 : PG_END_TRY();
5570 magnus@hagander.net 1068 : 175 : }
1069 : :
1070 : : /*
1071 : : * Inject a file with given name and content in the output tar stream.
1072 : : *
1073 : : * "len" can optionally be set to an arbitrary length of data sent. If set
1074 : : * to -1, the content sent is treated as a string with strlen() as length.
1075 : : */
1076 : : static void
1642 rhaas@postgresql.org 1077 : 238 : sendFileWithContent(bbsink *sink, const char *filename, const char *content,
1078 : : int len, backup_manifest_info *manifest)
1079 : : {
1080 : : struct stat statbuf;
904 michael@paquier.xyz 1081 : 238 : int bytes_done = 0;
1082 : : pg_checksum_context checksum_ctx;
1083 : :
1980 1084 [ - + ]: 238 : if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0)
1980 michael@paquier.xyz 1085 [ # # ]:UBC 0 : elog(ERROR, "could not initialize checksum of file \"%s\"",
1086 : : filename);
1087 : :
904 michael@paquier.xyz 1088 [ + - ]:CBC 238 : if (len < 0)
1089 : 238 : len = strlen(content);
1090 : :
1091 : : /*
1092 : : * Construct a stat struct for the file we're injecting in the tar.
1093 : : */
1094 : :
1095 : : /* Windows doesn't have the concept of uid and gid */
1096 : : #ifdef WIN32
1097 : : statbuf.st_uid = 0;
1098 : : statbuf.st_gid = 0;
1099 : : #else
5573 heikki.linnakangas@i 1100 : 238 : statbuf.st_uid = geteuid();
1101 : 238 : statbuf.st_gid = getegid();
1102 : : #endif
1103 : 238 : statbuf.st_mtime = time(NULL);
2950 sfrost@snowman.net 1104 : 238 : statbuf.st_mode = pg_file_create_mode;
5573 heikki.linnakangas@i 1105 : 238 : statbuf.st_size = len;
1106 : :
1642 rhaas@postgresql.org 1107 : 238 : _tarWriteHeader(sink, filename, NULL, &statbuf, false);
1108 : :
99 peter@eisentraut.org 1109 [ - + ]:GNC 238 : if (pg_checksum_update(&checksum_ctx, (const uint8 *) content, len) < 0)
1642 rhaas@postgresql.org 1110 [ # # ]:UBC 0 : elog(ERROR, "could not update checksum of file \"%s\"",
1111 : : filename);
1112 : :
1642 rhaas@postgresql.org 1113 [ + + ]:CBC 429 : while (bytes_done < len)
1114 : : {
1115 : 191 : size_t remaining = len - bytes_done;
1116 : 191 : size_t nbytes = Min(sink->bbs_buffer_length, remaining);
1117 : :
1118 : 191 : memcpy(sink->bbs_buffer, content, nbytes);
1119 : 191 : bbsink_archive_contents(sink, nbytes);
1120 : 191 : bytes_done += nbytes;
1240 1121 : 191 : content += nbytes;
1122 : : }
1123 : :
1642 1124 : 238 : _tarWritePadding(sink, len);
1125 : :
925 1126 : 238 : AddFileToBackupManifest(manifest, InvalidOid, filename, len,
2203 1127 : 238 : (pg_time_t) statbuf.st_mtime, &checksum_ctx);
5573 heikki.linnakangas@i 1128 : 238 : }
1129 : :
1130 : : /*
1131 : : * Include the tablespace directory pointed to by 'path' in the output tar
1132 : : * stream. If 'sizeonly' is true, we just calculate a total length and return
1133 : : * it, without actually sending anything.
1134 : : *
1135 : : * Only used to send auxiliary tablespaces, not PGDATA.
1136 : : */
1137 : : static int64
925 rhaas@postgresql.org 1138 : 78 : sendTablespace(bbsink *sink, char *path, Oid spcoid, bool sizeonly,
1139 : : backup_manifest_info *manifest, IncrementalBackupInfo *ib)
1140 : : {
1141 : : int64 size;
1142 : : char pathbuf[MAXPGPATH];
1143 : : struct stat statbuf;
1144 : :
1145 : : /*
1146 : : * 'path' points to the tablespace location, but we only want to include
1147 : : * the version directory in it that belongs to us.
1148 : : */
4789 heikki.linnakangas@i 1149 : 78 : snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path,
1150 : : TABLESPACE_VERSION_DIRECTORY);
1151 : :
1152 : : /*
1153 : : * Store a directory entry in the tar file so we get the permissions
1154 : : * right.
1155 : : */
1156 [ - + ]: 78 : if (lstat(pathbuf, &statbuf) != 0)
1157 : : {
4789 heikki.linnakangas@i 1158 [ # # ]:UBC 0 : if (errno != ENOENT)
1159 [ # # ]: 0 : ereport(ERROR,
1160 : : (errcode_for_file_access(),
1161 : : errmsg("could not stat file or directory \"%s\": %m",
1162 : : pathbuf)));
1163 : :
1164 : : /* If the tablespace went away while scanning, it's no error. */
1165 : 0 : return 0;
1166 : : }
1167 : :
1642 rhaas@postgresql.org 1168 :CBC 78 : size = _tarWriteHeader(sink, TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf,
1169 : : sizeonly);
1170 : :
1171 : : /* Send all the files in the tablespace version directory */
1172 : 78 : size += sendDir(sink, pathbuf, strlen(path), sizeonly, NIL, true, manifest,
1173 : : spcoid, ib);
1174 : :
4789 heikki.linnakangas@i 1175 : 78 : return size;
1176 : : }
1177 : :
1178 : : /*
1179 : : * Include all files from the given directory in the output tar stream. If
1180 : : * 'sizeonly' is true, we just calculate a total length and return it, without
1181 : : * actually sending anything.
1182 : : *
1183 : : * Omit any directory in the tablespaces list, to avoid backing up
1184 : : * tablespaces twice when they were created inside PGDATA.
1185 : : *
1186 : : * If sendtblspclinks is true, we need to include symlink
1187 : : * information in the tar file. If not, we can skip that
1188 : : * as it will be sent separately in the tablespace_map file.
1189 : : */
1190 : : static int64
1642 rhaas@postgresql.org 1191 : 6300 : sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
1192 : : List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest,
1193 : : Oid spcoid, IncrementalBackupInfo *ib)
1194 : : {
1195 : : DIR *dir;
1196 : : struct dirent *de;
1197 : : char pathbuf[MAXPGPATH * 2];
1198 : : struct stat statbuf;
5594 magnus@hagander.net 1199 : 6300 : int64 size = 0;
1200 : : const char *lastDir; /* Split last dir from parent path. */
903 rhaas@postgresql.org 1201 : 6300 : bool isRelationDir = false; /* Does directory contain relations? */
867 1202 : 6300 : bool isGlobalDir = false;
903 1203 : 6300 : Oid dboid = InvalidOid;
867 1204 : 6300 : BlockNumber *relative_block_numbers = NULL;
1205 : :
1206 : : /*
1207 : : * Since this array is relatively large, avoid putting it on the stack.
1208 : : * But we don't need it at all if this is not an incremental backup.
1209 : : */
1210 [ + + ]: 6300 : if (ib != NULL)
146 michael@paquier.xyz 1211 :GNC 191 : relative_block_numbers = palloc_array(BlockNumber, RELSEG_SIZE);
1212 : :
1213 : : /*
1214 : : * Determine if the current path is a database directory that can contain
1215 : : * relations.
1216 : : *
1217 : : * Start by finding the location of the delimiter between the parent path
1218 : : * and the current path.
1219 : : */
2965 teodor@sigaev.ru 1220 :CBC 6300 : lastDir = last_dir_separator(path);
1221 : :
1222 : : /* Does this path look like a database path (i.e. all digits)? */
1223 [ + + ]: 6300 : if (lastDir != NULL &&
1224 [ + + ]: 5934 : strspn(lastDir + 1, "0123456789") == strlen(lastDir + 1))
1225 : 1174 : {
1226 : : /* Part of path that contains the parent directory. */
2931 tgl@sss.pgh.pa.us 1227 : 1174 : int parentPathLen = lastDir - path;
1228 : :
1229 : : /*
1230 : : * Mark path as a database directory if the parent path is either
1231 : : * $PGDATA/base or a tablespace version path.
1232 : : */
2965 teodor@sigaev.ru 1233 [ + + ]: 1174 : if (strncmp(path, "./base", parentPathLen) == 0 ||
1234 [ + - ]: 56 : (parentPathLen >= (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) &&
1235 [ + - ]: 56 : strncmp(lastDir - (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1),
1236 : : TABLESPACE_VERSION_DIRECTORY,
1237 : : sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) == 0))
1238 : : {
903 rhaas@postgresql.org 1239 : 1174 : isRelationDir = true;
1240 : 1174 : dboid = atooid(lastDir + 1);
1241 : : }
1242 : : }
1243 [ + + ]: 5126 : else if (strcmp(path, "./global") == 0)
1244 : : {
1245 : 365 : isRelationDir = true;
867 1246 : 365 : isGlobalDir = true;
1247 : : }
1248 : :
5594 magnus@hagander.net 1249 : 6300 : dir = AllocateDir(path);
1250 [ + + ]: 413195 : while ((de = ReadDir(dir, path)) != NULL)
1251 : : {
1252 : : int excludeIdx;
1253 : : bool excludeFound;
903 rhaas@postgresql.org 1254 : 406905 : RelFileNumber relfilenumber = InvalidRelFileNumber;
1255 : 406905 : ForkNumber relForkNum = InvalidForkNumber;
1256 : 406905 : unsigned segno = 0;
1257 : 406905 : bool isRelationFile = false;
1258 : :
1259 : : /* Skip special stuff */
5594 magnus@hagander.net 1260 [ + + + + ]: 406905 : if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
1261 : 17811 : continue;
1262 : :
1263 : : /* Skip temporary files */
heikki.linnakangas@i 1264 [ + + ]: 394323 : if (strncmp(de->d_name,
1265 : : PG_TEMP_FILE_PREFIX,
1266 : : strlen(PG_TEMP_FILE_PREFIX)) == 0)
1267 : 361 : continue;
1268 : :
1269 : : /* Skip macOS system files */
812 dgustafsson@postgres 1270 [ + + ]: 393962 : if (strcmp(de->d_name, ".DS_Store") == 0)
1271 : 70 : continue;
1272 : :
1273 : : /*
1274 : : * Check if the postmaster has signaled us to exit, and abort with an
1275 : : * error in that case. The error handler further up will call
1276 : : * do_pg_abort_backup() for us. Also check that if the backup was
1277 : : * started while still in recovery, the server wasn't promoted.
1278 : : * do_pg_backup_stop() will check that too, but it's better to stop
1279 : : * the backup early than continue to the end and fail there.
1280 : : */
4891 heikki.linnakangas@i 1281 [ + + ]: 393892 : CHECK_FOR_INTERRUPTS();
1282 [ - + ]: 393888 : if (RecoveryInProgress() != backup_started_in_recovery)
5590 magnus@hagander.net 1283 [ # # ]:UBC 0 : ereport(ERROR,
1284 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1285 : : errmsg("the standby was promoted during online backup"),
1286 : : errhint("This means that the backup being taken is corrupt "
1287 : : "and should not be used. "
1288 : : "Try taking another online backup.")));
1289 : :
1290 : : /* Scan for files that should be excluded */
3506 peter_e@gmx.net 1291 :CBC 393888 : excludeFound = false;
2262 michael@paquier.xyz 1292 [ + + ]: 3539552 : for (excludeIdx = 0; excludeFiles[excludeIdx].name != NULL; excludeIdx++)
1293 : : {
1294 : 3147095 : int cmplen = strlen(excludeFiles[excludeIdx].name);
1295 : :
1296 [ + + ]: 3147095 : if (!excludeFiles[excludeIdx].match_prefix)
1297 : 2753337 : cmplen++;
1298 [ + + ]: 3147095 : if (strncmp(de->d_name, excludeFiles[excludeIdx].name, cmplen) == 0)
1299 : : {
3506 peter_e@gmx.net 1300 [ + + ]: 1431 : elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name);
1301 : 1431 : excludeFound = true;
1302 : 1431 : break;
1303 : : }
1304 : : }
1305 : :
1306 [ + + ]: 393888 : if (excludeFound)
5594 magnus@hagander.net 1307 : 1431 : continue;
1308 : :
1309 : : /*
1310 : : * If there could be non-temporary relation files in this directory,
1311 : : * try to parse the filename.
1312 : : */
903 rhaas@postgresql.org 1313 [ + + ]: 392457 : if (isRelationDir)
1314 : : isRelationFile =
1315 : 379815 : parse_filename_for_nontemp_relation(de->d_name,
1316 : : &relfilenumber,
1317 : : &relForkNum, &segno);
1318 : :
1319 : : /* Exclude all forks for unlogged tables except the init fork */
1320 [ + + + + ]: 392457 : if (isRelationFile && relForkNum != INIT_FORKNUM)
1321 : : {
1322 : : char initForkFile[MAXPGPATH];
1323 : :
1324 : : /*
1325 : : * If any other type of fork, check if there is an init fork with
1326 : : * the same RelFileNumber. If so, the file can be excluded.
1327 : : */
1328 : 376748 : snprintf(initForkFile, sizeof(initForkFile), "%s/%u_init",
1329 : : path, relfilenumber);
1330 : :
1331 [ + + ]: 376748 : if (lstat(initForkFile, &statbuf) == 0)
1332 : : {
1333 [ - + ]: 69 : elog(DEBUG2,
1334 : : "unlogged relation file \"%s\" excluded from backup",
1335 : : de->d_name);
1336 : :
1337 : 69 : continue;
1338 : : }
1339 : : }
1340 : :
1341 : : /* Exclude temporary relations */
1342 [ + + + + ]: 392388 : if (OidIsValid(dboid) && looks_like_temp_rel_name(de->d_name))
1343 : : {
2961 teodor@sigaev.ru 1344 [ - + ]: 36 : elog(DEBUG2,
1345 : : "temporary relation file \"%s\" excluded from backup",
1346 : : de->d_name);
1347 : :
1348 : 36 : continue;
1349 : : }
1350 : :
3311 peter_e@gmx.net 1351 : 392352 : snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name);
1352 : :
1353 : : /* Skip pg_control here to back up it last */
393 fujii@postgresql.org 1354 [ + + ]: 392352 : if (strcmp(pathbuf, "./" XLOG_CONTROL_FILE) == 0)
5214 simon@2ndQuadrant.co 1355 : 362 : continue;
1356 : :
5594 magnus@hagander.net 1357 [ - + ]: 391990 : if (lstat(pathbuf, &statbuf) != 0)
1358 : : {
5594 magnus@hagander.net 1359 [ # # ]:UBC 0 : if (errno != ENOENT)
1360 [ # # ]: 0 : ereport(ERROR,
1361 : : (errcode_for_file_access(),
1362 : : errmsg("could not stat file or directory \"%s\": %m",
1363 : : pathbuf)));
1364 : :
1365 : : /* If the file went away while scanning, it's not an error. */
1366 : 0 : continue;
1367 : : }
1368 : :
1369 : : /* Scan for directories whose contents should be excluded */
3506 peter_e@gmx.net 1370 :CBC 391990 : excludeFound = false;
1371 [ + + ]: 3125776 : for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)
1372 : : {
1373 [ + + ]: 2736321 : if (strcmp(de->d_name, excludeDirContents[excludeIdx]) == 0)
1374 : : {
1375 [ + + ]: 2535 : elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name);
1666 rhaas@postgresql.org 1376 : 2535 : convert_link_to_directory(pathbuf, &statbuf);
1642 1377 : 2535 : size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,
1378 : : &statbuf, sizeonly);
3506 peter_e@gmx.net 1379 : 2535 : excludeFound = true;
1380 : 2535 : break;
1381 : : }
1382 : : }
1383 : :
1384 [ + + ]: 391990 : if (excludeFound)
1385 : 2535 : continue;
1386 : :
1387 : : /*
1388 : : * We can skip pg_wal, the WAL segments need to be fetched from the
1389 : : * WAL archive anyway. But include it as an empty directory anyway, so
1390 : : * we get permissions right.
1391 : : */
3484 rhaas@postgresql.org 1392 [ + + ]: 389455 : if (strcmp(pathbuf, "./pg_wal") == 0)
1393 : : {
1394 : : /* If pg_wal is a symlink, write it as a directory anyway */
1666 1395 : 365 : convert_link_to_directory(pathbuf, &statbuf);
1642 1396 : 365 : size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,
1397 : : &statbuf, sizeonly);
1398 : :
1399 : : /*
1400 : : * Also send archive_status and summaries directories (by
1401 : : * hackishly reusing statbuf from above ...).
1402 : : */
1403 : 365 : size += _tarWriteHeader(sink, "./pg_wal/archive_status", NULL,
1404 : : &statbuf, sizeonly);
867 1405 : 365 : size += _tarWriteHeader(sink, "./pg_wal/summaries", NULL,
1406 : : &statbuf, sizeonly);
1407 : :
3484 1408 : 365 : continue; /* don't recurse into pg_wal */
1409 : : }
1410 : :
1411 : : /* Allow symbolic links in pg_tblspc only */
1368 tmunro@postgresql.or 1412 [ + + + + ]: 389090 : if (strcmp(path, "./pg_tblspc") == 0 && S_ISLNK(statbuf.st_mode))
5594 magnus@hagander.net 1413 : 41 : {
1414 : : char linkpath[MAXPGPATH];
1415 : : int rllen;
1416 : :
5263 tgl@sss.pgh.pa.us 1417 : 41 : rllen = readlink(pathbuf, linkpath, sizeof(linkpath));
1418 [ - + ]: 41 : if (rllen < 0)
5594 magnus@hagander.net 1419 [ # # ]:UBC 0 : ereport(ERROR,
1420 : : (errcode_for_file_access(),
1421 : : errmsg("could not read symbolic link \"%s\": %m",
1422 : : pathbuf)));
5263 tgl@sss.pgh.pa.us 1423 [ - + ]:CBC 41 : if (rllen >= sizeof(linkpath))
5263 tgl@sss.pgh.pa.us 1424 [ # # ]:UBC 0 : ereport(ERROR,
1425 : : (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1426 : : errmsg("symbolic link \"%s\" target is too long",
1427 : : pathbuf)));
5263 tgl@sss.pgh.pa.us 1428 :CBC 41 : linkpath[rllen] = '\0';
1429 : :
1642 rhaas@postgresql.org 1430 : 41 : size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, linkpath,
1431 : : &statbuf, sizeonly);
1432 : : }
5594 magnus@hagander.net 1433 [ + + ]: 389049 : else if (S_ISDIR(statbuf.st_mode))
1434 : : {
4501 1435 : 5916 : bool skip_this_dir = false;
1436 : : ListCell *lc;
1437 : :
1438 : : /*
1439 : : * Store a directory entry in the tar file so we can get the
1440 : : * permissions right.
1441 : : */
1642 rhaas@postgresql.org 1442 : 5916 : size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL, &statbuf,
1443 : : sizeonly);
1444 : :
1445 : : /*
1446 : : * Call ourselves recursively for a directory, unless it happens
1447 : : * to be a separate tablespace located within PGDATA.
1448 : : */
4501 magnus@hagander.net 1449 [ + + + + : 13025 : foreach(lc, tablespaces)
+ + ]
1450 : : {
1451 : 7137 : tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
1452 : :
1453 : : /*
1454 : : * ti->rpath is the tablespace relative path within PGDATA, or
1455 : : * NULL if the tablespace has been properly located somewhere
1456 : : * else.
1457 : : *
1458 : : * Skip past the leading "./" in pathbuf when comparing.
1459 : : */
1460 [ + + + + ]: 7137 : if (ti->rpath && strcmp(ti->rpath, pathbuf + 2) == 0)
1461 : : {
1462 : 28 : skip_this_dir = true;
1463 : 28 : break;
1464 : : }
1465 : : }
1466 : :
1467 : : /*
1468 : : * skip sending directories inside pg_tblspc, if not required.
1469 : : */
4011 andrew@dunslane.net 1470 [ + + + + ]: 5916 : if (strcmp(pathbuf, "./pg_tblspc") == 0 && !sendtblspclinks)
1471 : 32 : skip_this_dir = true;
1472 : :
4501 magnus@hagander.net 1473 [ + + ]: 5916 : if (!skip_this_dir)
1642 rhaas@postgresql.org 1474 : 5856 : size += sendDir(sink, pathbuf, basepathlen, sizeonly, tablespaces,
1475 : : sendtblspclinks, manifest, spcoid, ib);
1476 : : }
5594 magnus@hagander.net 1477 [ + - ]: 383133 : else if (S_ISREG(statbuf.st_mode))
1478 : : {
4724 bruce@momjian.us 1479 : 383133 : bool sent = false;
867 rhaas@postgresql.org 1480 : 383133 : unsigned num_blocks_required = 0;
1481 : 383133 : unsigned truncation_block_length = 0;
1482 : : char tarfilenamebuf[MAXPGPATH * 2];
1483 : 383133 : char *tarfilename = pathbuf + basepathlen + 1;
1484 : 383133 : FileBackupMethod method = BACK_UP_FILE_FULLY;
1485 : :
1486 [ + + + + ]: 383133 : if (ib != NULL && isRelationFile)
1487 : : {
1488 : : Oid relspcoid;
1489 : : char *lookup_path;
1490 : :
1491 [ + + ]: 11795 : if (OidIsValid(spcoid))
1492 : : {
1493 : 9 : relspcoid = spcoid;
609 michael@paquier.xyz 1494 : 9 : lookup_path = psprintf("%s/%u/%s", PG_TBLSPC_DIR, spcoid,
1495 : : tarfilename);
1496 : : }
1497 : : else
1498 : : {
867 rhaas@postgresql.org 1499 [ + + ]: 11786 : if (isGlobalDir)
1500 : 616 : relspcoid = GLOBALTABLESPACE_OID;
1501 : : else
1502 : 11170 : relspcoid = DEFAULTTABLESPACE_OID;
1503 : 11786 : lookup_path = pstrdup(tarfilename);
1504 : : }
1505 : :
1506 : 11795 : method = GetFileBackupMethod(ib, lookup_path, dboid, relspcoid,
1507 : : relfilenumber, relForkNum,
1508 : 11795 : segno, statbuf.st_size,
1509 : : &num_blocks_required,
1510 : : relative_block_numbers,
1511 : : &truncation_block_length);
1512 [ + + ]: 11795 : if (method == BACK_UP_FILE_INCREMENTALLY)
1513 : : {
1514 : 7756 : statbuf.st_size =
1515 : 7756 : GetIncrementalFileSize(num_blocks_required);
1516 : 7756 : snprintf(tarfilenamebuf, sizeof(tarfilenamebuf),
1517 : : "%s/INCREMENTAL.%s",
1518 : 7756 : path + basepathlen + 1,
1519 : 7756 : de->d_name);
1520 : 7756 : tarfilename = tarfilenamebuf;
1521 : : }
1522 : :
1523 : 11795 : pfree(lookup_path);
1524 : : }
1525 : :
5594 magnus@hagander.net 1526 [ + + ]: 383133 : if (!sizeonly)
867 rhaas@postgresql.org 1527 [ + + ]: 189060 : sent = sendFile(sink, pathbuf, tarfilename, &statbuf,
1528 : : true, dboid, spcoid,
1529 : : relfilenumber, segno, manifest,
1530 : : num_blocks_required,
1531 : : method == BACK_UP_FILE_INCREMENTALLY ? relative_block_numbers : NULL,
1532 : : truncation_block_length);
1533 : :
4883 heikki.linnakangas@i 1534 [ + + + - ]: 383132 : if (sent || sizeonly)
1535 : : {
1536 : : /* Add size. */
2202 rhaas@postgresql.org 1537 : 383132 : size += statbuf.st_size;
1538 : :
1539 : : /* Pad to a multiple of the tar block size. */
1540 : 383132 : size += tarPaddingBytesRequired(statbuf.st_size);
1541 : :
1542 : : /* Size of the header for the file. */
1543 : 383132 : size += TAR_BLOCK_SIZE;
1544 : : }
1545 : : }
1546 : : else
5594 magnus@hagander.net 1547 [ # # ]:UBC 0 : ereport(WARNING,
1548 : : (errmsg("skipping special file \"%s\"", pathbuf)));
1549 : : }
1550 : :
867 rhaas@postgresql.org 1551 [ + + ]:CBC 6290 : if (relative_block_numbers != NULL)
1552 : 191 : pfree(relative_block_numbers);
1553 : :
5594 magnus@hagander.net 1554 : 6290 : FreeDir(dir);
1555 : 6290 : return size;
1556 : : }
1557 : :
1558 : : /*
1559 : : * Given the member, write the TAR header & send the file.
1560 : : *
1561 : : * If 'missing_ok' is true, will not throw an error if the file is not found.
1562 : : *
1563 : : * If dboid is anything other than InvalidOid then any checksum failures
1564 : : * detected will get reported to the cumulative stats system.
1565 : : *
1566 : : * If the file is to be sent incrementally, then num_incremental_blocks
1567 : : * should be the number of blocks to be sent, and incremental_blocks
1568 : : * an array of block numbers relative to the start of the current segment.
1569 : : * If the whole file is to be sent, then incremental_blocks should be NULL,
1570 : : * and num_incremental_blocks can have any value, as it will be ignored.
1571 : : *
1572 : : * Returns true if the file was successfully sent, false if 'missing_ok',
1573 : : * and the file did not exist.
1574 : : */
1575 : : static bool
1642 rhaas@postgresql.org 1576 : 189238 : sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
1577 : : struct stat *statbuf, bool missing_ok, Oid dboid, Oid spcoid,
1578 : : RelFileNumber relfilenumber, unsigned segno,
1579 : : backup_manifest_info *manifest, unsigned num_incremental_blocks,
1580 : : BlockNumber *incremental_blocks, unsigned truncation_block_length)
1581 : : {
1582 : : int fd;
2954 magnus@hagander.net 1583 : 189238 : BlockNumber blkno = 0;
1584 : 189238 : int checksum_failures = 0;
1585 : : off_t cnt;
945 rhaas@postgresql.org 1586 : 189238 : pgoff_t bytes_done = 0;
2954 magnus@hagander.net 1587 : 189238 : bool verify_checksum = false;
1588 : : pg_checksum_context checksum_ctx;
867 rhaas@postgresql.org 1589 : 189238 : int ibindex = 0;
1590 : :
1980 michael@paquier.xyz 1591 [ - + ]: 189238 : if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0)
1980 michael@paquier.xyz 1592 [ # # ]:UBC 0 : elog(ERROR, "could not initialize checksum of file \"%s\"",
1593 : : readfilename);
1594 : :
2148 rhaas@postgresql.org 1595 :CBC 189238 : fd = OpenTransientFile(readfilename, O_RDONLY | PG_BINARY);
1596 [ - + ]: 189238 : if (fd < 0)
1597 : : {
4883 heikki.linnakangas@i 1598 [ # # # # ]:UBC 0 : if (errno == ENOENT && missing_ok)
1599 : 0 : return false;
5594 magnus@hagander.net 1600 [ # # ]: 0 : ereport(ERROR,
1601 : : (errcode_for_file_access(),
1602 : : errmsg("could not open file \"%s\": %m", readfilename)));
1603 : : }
1604 : :
1642 rhaas@postgresql.org 1605 :CBC 189238 : _tarWriteHeader(sink, tarfilename, NULL, statbuf, false);
1606 : :
1607 : : /*
1608 : : * Checksums are verified in multiples of BLCKSZ, so the buffer length
1609 : : * should be a multiple of the block size as well.
1610 : : */
945 1611 [ - + ]: 189237 : Assert((sink->bbs_buffer_length % BLCKSZ) == 0);
1612 : :
1613 : : /*
1614 : : * If we weren't told not to verify checksums, and if checksums are
1615 : : * enabled for this cluster, and if this is a relation file, then verify
1616 : : * the checksum. We cannot at this point check if checksums are enabled
1617 : : * or disabled as that might change, thus we check at each point where we
1618 : : * could be validating a checksum.
1619 : : */
32 dgustafsson@postgres 1620 [ + + + + ]:GNC 189237 : if (!noverify_checksums && RelFileNumberIsValid(relfilenumber))
903 rhaas@postgresql.org 1621 :CBC 184896 : verify_checksum = true;
1622 : :
1623 : : /*
1624 : : * If we're sending an incremental file, write the file header.
1625 : : */
867 1626 [ + + ]: 189237 : if (incremental_blocks != NULL)
1627 : : {
1628 : 7756 : unsigned magic = INCREMENTAL_MAGIC;
1629 : 7756 : size_t header_bytes_done = 0;
1630 : : char padding[BLCKSZ];
1631 : : size_t paddinglen;
1632 : :
1633 : : /* Emit header data. */
1634 : 7756 : push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1635 : : &magic, sizeof(magic));
1636 : 7756 : push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1637 : : &num_incremental_blocks, sizeof(num_incremental_blocks));
1638 : 7756 : push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1639 : : &truncation_block_length, sizeof(truncation_block_length));
1640 : 7756 : push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1641 : : incremental_blocks,
1642 : : sizeof(BlockNumber) * num_incremental_blocks);
1643 : :
1644 : : /*
1645 : : * Add padding to align header to a multiple of BLCKSZ, but only if
1646 : : * the incremental file has some blocks, and the alignment is actually
1647 : : * needed (i.e. header is not already a multiple of BLCKSZ). If there
1648 : : * are no blocks we don't want to make the file unnecessarily large,
1649 : : * as that might make some filesystem optimizations impossible.
1650 : : */
751 tomas.vondra@postgre 1651 [ + + + - ]: 7756 : if ((num_incremental_blocks > 0) && (header_bytes_done % BLCKSZ != 0))
1652 : : {
760 1653 : 25 : paddinglen = (BLCKSZ - (header_bytes_done % BLCKSZ));
1654 : :
1655 : 25 : memset(padding, 0, paddinglen);
1656 : 25 : bytes_done += paddinglen;
1657 : :
1658 : 25 : push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1659 : : padding, paddinglen);
1660 : : }
1661 : :
1662 : : /* Flush out any data still in the buffer so it's again empty. */
867 rhaas@postgresql.org 1663 [ + - ]: 7756 : if (header_bytes_done > 0)
1664 : : {
1665 : 7756 : bbsink_archive_contents(sink, header_bytes_done);
1666 [ - + ]: 7756 : if (pg_checksum_update(&checksum_ctx,
1667 : 7756 : (uint8 *) sink->bbs_buffer,
1668 : : header_bytes_done) < 0)
867 rhaas@postgresql.org 1669 [ # # ]:UBC 0 : elog(ERROR, "could not update checksum of base backup");
1670 : : }
1671 : :
1672 : : /* Update our notion of file position. */
867 rhaas@postgresql.org 1673 :CBC 7756 : bytes_done += sizeof(magic);
1674 : 7756 : bytes_done += sizeof(num_incremental_blocks);
1675 : 7756 : bytes_done += sizeof(truncation_block_length);
1676 : 7756 : bytes_done += sizeof(BlockNumber) * num_incremental_blocks;
1677 : : }
1678 : :
1679 : : /*
1680 : : * Loop until we read the amount of data the caller told us to expect. The
1681 : : * file could be longer, if it was extended while we were sending it, but
1682 : : * for a base backup we can ignore such extended data. It will be restored
1683 : : * from WAL.
1684 : : */
1685 : : while (1)
1686 : : {
1687 : : /*
1688 : : * Determine whether we've read all the data that we need, and if not,
1689 : : * read some more.
1690 : : */
1691 [ + + ]: 406399 : if (incremental_blocks == NULL)
1692 : : {
1693 : 398606 : size_t remaining = statbuf->st_size - bytes_done;
1694 : :
1695 : : /*
1696 : : * If we've read the required number of bytes, then it's time to
1697 : : * stop.
1698 : : */
1699 [ + + ]: 398606 : if (bytes_done >= statbuf->st_size)
1700 : 181481 : break;
1701 : :
1702 : : /*
1703 : : * Read as many bytes as will fit in the buffer, or however many
1704 : : * are left to read, whichever is less.
1705 : : */
1706 : 217125 : cnt = read_file_data_into_buffer(sink, readfilename, fd,
1707 : : bytes_done, remaining,
1708 : 217125 : blkno + segno * RELSEG_SIZE,
1709 : : verify_checksum,
1710 : : &checksum_failures);
1711 : : }
1712 : : else
1713 : : {
1714 : : BlockNumber relative_blkno;
1715 : :
1716 : : /*
1717 : : * If we've read all the blocks, then it's time to stop.
1718 : : */
1719 [ + + ]: 7793 : if (ibindex >= num_incremental_blocks)
1720 : 7756 : break;
1721 : :
1722 : : /*
1723 : : * Read just one block, whichever one is the next that we're
1724 : : * supposed to include.
1725 : : */
1726 : 37 : relative_blkno = incremental_blocks[ibindex++];
1727 : 37 : cnt = read_file_data_into_buffer(sink, readfilename, fd,
1728 : 37 : relative_blkno * BLCKSZ,
1729 : : BLCKSZ,
1730 : 37 : relative_blkno + segno * RELSEG_SIZE,
1731 : : verify_checksum,
1732 : : &checksum_failures);
1733 : :
1734 : : /*
1735 : : * If we get a partial read, that must mean that the relation is
1736 : : * being truncated. Ultimately, it should be truncated to a
1737 : : * multiple of BLCKSZ, since this path should only be reached for
1738 : : * relation files, but we might transiently observe an
1739 : : * intermediate value.
1740 : : *
1741 : : * It should be fine to treat this just as if the entire block had
1742 : : * been truncated away - i.e. fill this and all later blocks with
1743 : : * zeroes. WAL replay will fix things up.
1744 : : */
1745 [ - + ]: 37 : if (cnt < BLCKSZ)
867 rhaas@postgresql.org 1746 :UBC 0 : break;
1747 : : }
1748 : :
1749 : : /*
1750 : : * If the amount of data we were able to read was not a multiple of
1751 : : * BLCKSZ, we cannot verify checksums, which are block-level.
1752 : : */
32 dgustafsson@postgres 1753 [ + + + + :GNC 217162 : if (verify_checksum && DataChecksumsNeedVerify() && (cnt % BLCKSZ != 0))
- + ]
1754 : : {
2954 magnus@hagander.net 1755 [ # # ]:UBC 0 : ereport(WARNING,
1756 : : (errmsg("could not verify checksum in file \"%s\", block "
1757 : : "%u: read buffer size %d and page size %d "
1758 : : "differ",
1759 : : readfilename, blkno, (int) cnt, BLCKSZ)));
1760 : 0 : verify_checksum = false;
1761 : : }
1762 : :
1763 : : /*
1764 : : * If we hit end-of-file, a concurrent truncation must have occurred.
1765 : : * That's not an error condition, because WAL replay will fix things
1766 : : * up.
1767 : : */
1188 rhaas@postgresql.org 1768 [ - + ]:CBC 217162 : if (cnt == 0)
1188 rhaas@postgresql.org 1769 :UBC 0 : break;
1770 : :
1771 : : /* Update block number and # of bytes done for next loop iteration. */
945 rhaas@postgresql.org 1772 :CBC 217162 : blkno += cnt / BLCKSZ;
1773 : 217162 : bytes_done += cnt;
1774 : :
1775 : : /*
1776 : : * Make sure incremental files with block data are properly aligned
1777 : : * (header is a multiple of BLCKSZ, blocks are BLCKSZ too).
1778 : : */
760 tomas.vondra@postgre 1779 [ + + + - : 217162 : Assert(!((incremental_blocks != NULL && num_incremental_blocks > 0) &&
- + ]
1780 : : (bytes_done % BLCKSZ != 0)));
1781 : :
1782 : : /* Archive the data we just read. */
1642 rhaas@postgresql.org 1783 : 217162 : bbsink_archive_contents(sink, cnt);
1784 : :
1785 : : /* Also feed it to the checksum machinery. */
1786 [ - + ]: 217162 : if (pg_checksum_update(&checksum_ctx,
1787 : 217162 : (uint8 *) sink->bbs_buffer, cnt) < 0)
1980 michael@paquier.xyz 1788 [ # # ]:UBC 0 : elog(ERROR, "could not update checksum of base backup");
1789 : : }
1790 : :
1791 : : /* If the file was truncated while we were sending it, pad it with zeros */
945 rhaas@postgresql.org 1792 [ - + ]:CBC 189237 : while (bytes_done < statbuf->st_size)
1793 : : {
945 rhaas@postgresql.org 1794 :UBC 0 : size_t remaining = statbuf->st_size - bytes_done;
1642 1795 : 0 : size_t nbytes = Min(sink->bbs_buffer_length, remaining);
1796 : :
1797 [ # # # # : 0 : MemSet(sink->bbs_buffer, 0, nbytes);
# # # # #
# ]
1798 [ # # ]: 0 : if (pg_checksum_update(&checksum_ctx,
1799 : 0 : (uint8 *) sink->bbs_buffer,
1800 : : nbytes) < 0)
1801 [ # # ]: 0 : elog(ERROR, "could not update checksum of base backup");
1802 : 0 : bbsink_archive_contents(sink, nbytes);
945 1803 : 0 : bytes_done += nbytes;
1804 : : }
1805 : :
1806 : : /*
1807 : : * Pad to a block boundary, per tar format requirements. (This small piece
1808 : : * of data is probably not worth throttling, and is not checksummed
1809 : : * because it's not actually part of the file.)
1810 : : */
945 rhaas@postgresql.org 1811 :CBC 189237 : _tarWritePadding(sink, bytes_done);
1812 : :
2148 1813 : 189237 : CloseTransientFile(fd);
1814 : :
2954 magnus@hagander.net 1815 [ + + ]: 189237 : if (checksum_failures > 1)
1816 : : {
1817 [ + - ]: 2 : ereport(WARNING,
1818 : : (errmsg_plural("file \"%s\" has a total of %d checksum verification failure",
1819 : : "file \"%s\" has a total of %d checksum verification failures",
1820 : : checksum_failures,
1821 : : readfilename, checksum_failures)));
1822 : :
401 andres@anarazel.de 1823 : 2 : pgstat_prepare_report_checksum_failure(dboid);
2580 magnus@hagander.net 1824 : 2 : pgstat_report_checksum_failures_in_db(dboid, checksum_failures);
1825 : : }
1826 : :
2954 1827 : 189237 : total_checksum_failures += checksum_failures;
1828 : :
2203 rhaas@postgresql.org 1829 : 189237 : AddFileToBackupManifest(manifest, spcoid, tarfilename, statbuf->st_size,
1830 : 189237 : (pg_time_t) statbuf->st_mtime, &checksum_ctx);
1831 : :
4883 heikki.linnakangas@i 1832 : 189237 : return true;
1833 : : }
1834 : :
1835 : : /*
1836 : : * Read some more data from the file into the bbsink's buffer, verifying
1837 : : * checksums as required.
1838 : : *
1839 : : * 'offset' is the file offset from which we should begin to read, and
1840 : : * 'length' is the amount of data that should be read. The actual amount
1841 : : * of data read will be less than the requested amount if the bbsink's
1842 : : * buffer isn't big enough to hold it all, or if the underlying file has
1843 : : * been truncated. The return value is the number of bytes actually read.
1844 : : *
1845 : : * 'blkno' is the block number of the first page in the bbsink's buffer
1846 : : * relative to the start of the relation.
1847 : : *
1848 : : * 'verify_checksum' determines if the user has asked to verify checksums, but
1849 : : * since data checksums can be disabled, or become disabled, we need to check
1850 : : * state before verifying individual pages. If we do this, we'll update
1851 : : * *checksum_failures and issue warnings as appropriate.
1852 : : */
1853 : : static off_t
945 rhaas@postgresql.org 1854 : 217162 : read_file_data_into_buffer(bbsink *sink, const char *readfilename, int fd,
1855 : : off_t offset, size_t length, BlockNumber blkno,
1856 : : bool verify_checksum, int *checksum_failures)
1857 : : {
1858 : : off_t cnt;
1859 : : int i;
1860 : : char *page;
1861 : :
1862 : : /* Try to read some more data. */
1863 : 217162 : cnt = basebackup_read_file(fd, sink->bbs_buffer,
1864 : 217162 : Min(sink->bbs_buffer_length, length),
1865 : : offset, readfilename, true);
1866 : :
1867 : : /* Can't verify checksums if read length is not a multiple of BLCKSZ. */
1868 [ + + - + ]: 217162 : if (!verify_checksum || (cnt % BLCKSZ) != 0)
1869 : 4992 : return cnt;
1870 : :
1871 : : /* Verify checksum for each block. */
1872 [ + + ]: 732853 : for (i = 0; i < cnt / BLCKSZ; i++)
1873 : : {
1874 : : int reread_cnt;
1875 : : uint16 expected_checksum;
1876 : :
1877 : : /*
1878 : : * The data checksum state can change at any point, so we need to
1879 : : * re-check before each page.
1880 : : */
32 dgustafsson@postgres 1881 [ + + ]:GNC 521908 : if (!DataChecksumsNeedVerify())
1882 : 1225 : return cnt;
1883 : :
945 rhaas@postgresql.org 1884 :CBC 520683 : page = sink->bbs_buffer + BLCKSZ * i;
1885 : :
1886 : : /* If the page is OK, go on to the next one. */
1887 [ + + ]: 520683 : if (verify_page_checksum(page, sink->bbs_state->startptr, blkno + i,
1888 : : &expected_checksum))
1889 : 520669 : continue;
1890 : :
1891 : : /*
1892 : : * Retry the block on the first failure. It's possible that we read
1893 : : * the first 4K page of the block just before postgres updated the
1894 : : * entire block so it ends up looking torn to us. If, before we retry
1895 : : * the read, the concurrent write of the block finishes, the page LSN
1896 : : * will be updated and we'll realize that we should ignore this block.
1897 : : *
1898 : : * There's no guarantee that this will actually happen, though: the
1899 : : * torn write could take an arbitrarily long time to complete.
1900 : : * Retrying multiple times wouldn't fix this problem, either, though
1901 : : * it would reduce the chances of it happening in practice. The only
1902 : : * real fix here seems to be to have some kind of interlock that
1903 : : * allows us to wait until we can be certain that no write to the
1904 : : * block is in progress. Since we don't have any such thing right now,
1905 : : * we just do this and hope for the best.
1906 : : *
1907 : : * The data checksum state may also have changed concurrently so check
1908 : : * again.
1909 : : */
32 dgustafsson@postgres 1910 [ - + ]:GNC 14 : if (!DataChecksumsNeedVerify())
32 dgustafsson@postgres 1911 :UNC 0 : return cnt;
945 rhaas@postgresql.org 1912 :CBC 14 : reread_cnt =
1913 : 14 : basebackup_read_file(fd, sink->bbs_buffer + BLCKSZ * i,
1914 : 14 : BLCKSZ, offset + BLCKSZ * i,
1915 : : readfilename, false);
1916 [ - + ]: 14 : if (reread_cnt == 0)
1917 : : {
1918 : : /*
1919 : : * If we hit end-of-file, a concurrent truncation must have
1920 : : * occurred, so reduce cnt to reflect only the blocks already
1921 : : * processed and break out of this loop.
1922 : : */
945 rhaas@postgresql.org 1923 :UBC 0 : cnt = BLCKSZ * i;
1924 : 0 : break;
1925 : : }
1926 : :
1927 : : /* If the page now looks OK, go on to the next one. */
945 rhaas@postgresql.org 1928 [ - + ]:CBC 14 : if (verify_page_checksum(page, sink->bbs_state->startptr, blkno + i,
1929 : : &expected_checksum))
945 rhaas@postgresql.org 1930 :UBC 0 : continue;
1931 : :
1932 : : /* Handle checksum failure. */
945 rhaas@postgresql.org 1933 :CBC 14 : (*checksum_failures)++;
1934 [ + + ]: 14 : if (*checksum_failures <= 5)
1935 [ + - ]: 12 : ereport(WARNING,
1936 : : (errmsg("checksum verification failed in "
1937 : : "file \"%s\", block %u: calculated "
1938 : : "%X but expected %X",
1939 : : readfilename, blkno + i, expected_checksum,
1940 : : ((PageHeader) page)->pd_checksum)));
1941 [ + + ]: 14 : if (*checksum_failures == 5)
1942 [ + - ]: 2 : ereport(WARNING,
1943 : : (errmsg("further checksum verification "
1944 : : "failures in file \"%s\" will not "
1945 : : "be reported", readfilename)));
1946 : : }
1947 : :
1948 : 210945 : return cnt;
1949 : : }
1950 : :
1951 : : /*
1952 : : * Push data into a bbsink.
1953 : : *
1954 : : * It's better, when possible, to read data directly into the bbsink's buffer,
1955 : : * rather than using this function to copy it into the buffer; this function is
1956 : : * for cases where that approach is not practical.
1957 : : *
1958 : : * bytes_done should point to a count of the number of bytes that are
1959 : : * currently used in the bbsink's buffer. Upon return, the bytes identified by
1960 : : * data and length will have been copied into the bbsink's buffer, flushing
1961 : : * as required, and *bytes_done will have been updated accordingly. If the
1962 : : * buffer was flushed, the previous contents will also have been fed to
1963 : : * checksum_ctx.
1964 : : *
1965 : : * Note that after one or more calls to this function it is the caller's
1966 : : * responsibility to perform any required final flush.
1967 : : */
1968 : : static void
867 1969 : 31049 : push_to_sink(bbsink *sink, pg_checksum_context *checksum_ctx,
1970 : : size_t *bytes_done, void *data, size_t length)
1971 : : {
1972 [ + + ]: 31049 : while (length > 0)
1973 : : {
1974 : : size_t bytes_to_copy;
1975 : :
1976 : : /*
1977 : : * We use < here rather than <= so that if the data exactly fills the
1978 : : * remaining buffer space, we trigger a flush now.
1979 : : */
1980 [ + - ]: 23318 : if (length < sink->bbs_buffer_length - *bytes_done)
1981 : : {
1982 : : /* Append remaining data to buffer. */
1983 : 23318 : memcpy(sink->bbs_buffer + *bytes_done, data, length);
1984 : 23318 : *bytes_done += length;
1985 : 23318 : return;
1986 : : }
1987 : :
1988 : : /* Copy until buffer is full and flush it. */
867 rhaas@postgresql.org 1989 :UBC 0 : bytes_to_copy = sink->bbs_buffer_length - *bytes_done;
1990 : 0 : memcpy(sink->bbs_buffer + *bytes_done, data, bytes_to_copy);
1991 : 0 : data = ((char *) data) + bytes_to_copy;
1992 : 0 : length -= bytes_to_copy;
1993 : 0 : bbsink_archive_contents(sink, sink->bbs_buffer_length);
1994 [ # # ]: 0 : if (pg_checksum_update(checksum_ctx, (uint8 *) sink->bbs_buffer,
1995 : : sink->bbs_buffer_length) < 0)
1996 [ # # ]: 0 : elog(ERROR, "could not update checksum");
1997 : 0 : *bytes_done = 0;
1998 : : }
1999 : : }
2000 : :
2001 : : /*
2002 : : * Try to verify the checksum for the provided page, if it seems appropriate
2003 : : * to do so.
2004 : : *
2005 : : * Returns true if verification succeeds or if we decide not to check it,
2006 : : * and false if verification fails. When return false, it also sets
2007 : : * *expected_checksum to the computed value.
2008 : : */
2009 : : static bool
945 rhaas@postgresql.org 2010 :CBC 520697 : verify_page_checksum(Page page, XLogRecPtr start_lsn, BlockNumber blkno,
2011 : : uint16 *expected_checksum)
2012 : : {
2013 : : PageHeader phdr;
2014 : : uint16 checksum;
2015 : :
2016 : : /*
2017 : : * Only check pages which have not been modified since the start of the
2018 : : * base backup. Otherwise, they might have been written only halfway and
2019 : : * the checksum would not be valid. However, replaying WAL would
2020 : : * reinstate the correct page in this case. We also skip completely new
2021 : : * pages, since they don't have a checksum yet.
2022 : : */
2023 [ + + + + ]: 520697 : if (PageIsNew(page) || PageGetLSN(page) >= start_lsn)
2024 : 1073 : return true;
2025 : :
32 dgustafsson@postgres 2026 [ - + ]:GNC 519624 : if (!DataChecksumsNeedVerify())
32 dgustafsson@postgres 2027 :UNC 0 : return true;
2028 : :
2029 : : /* Perform the actual checksum calculation. */
945 rhaas@postgresql.org 2030 :CBC 519624 : checksum = pg_checksum_page(page, blkno);
2031 : :
2032 : : /* See whether it matches the value from the page. */
2033 : 519624 : phdr = (PageHeader) page;
2034 [ + + ]: 519624 : if (phdr->pd_checksum == checksum)
2035 : 519596 : return true;
2036 : 28 : *expected_checksum = checksum;
2037 : 28 : return false;
2038 : : }
2039 : :
2040 : : static int64
1642 2041 : 199163 : _tarWriteHeader(bbsink *sink, const char *filename, const char *linktarget,
2042 : : struct stat *statbuf, bool sizeonly)
2043 : : {
2044 : : enum tarError rc;
2045 : :
3506 peter_e@gmx.net 2046 [ + + ]: 199163 : if (!sizeonly)
2047 : : {
2048 : : /*
2049 : : * As of this writing, the smallest supported block size is 1kB, which
2050 : : * is twice TAR_BLOCK_SIZE. Since the buffer size is required to be a
2051 : : * multiple of BLCKSZ, it should be safe to assume that the buffer is
2052 : : * large enough to fit an entire tar block. We double-check by means
2053 : : * of these assertions.
2054 : : */
2055 : : StaticAssertDecl(TAR_BLOCK_SIZE <= BLCKSZ,
2056 : : "BLCKSZ too small for tar block");
1642 rhaas@postgresql.org 2057 [ - + ]: 194278 : Assert(sink->bbs_buffer_length >= TAR_BLOCK_SIZE);
2058 : :
2059 : 194278 : rc = tarCreateHeader(sink->bbs_buffer, filename, linktarget,
2060 : : statbuf->st_size, statbuf->st_mode,
2061 : : statbuf->st_uid, statbuf->st_gid,
2062 : : statbuf->st_mtime);
2063 : :
3506 peter_e@gmx.net 2064 [ + + - - ]: 194278 : switch (rc)
2065 : : {
2066 : 194277 : case TAR_OK:
2067 : 194277 : break;
2068 : 1 : case TAR_NAME_TOO_LONG:
2069 [ + - ]: 1 : ereport(ERROR,
2070 : : (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
2071 : : errmsg("file name too long for tar format: \"%s\"",
2072 : : filename)));
2073 : : break;
3506 peter_e@gmx.net 2074 :UBC 0 : case TAR_SYMLINK_TOO_LONG:
2075 [ # # ]: 0 : ereport(ERROR,
2076 : : (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
2077 : : errmsg("symbolic link target too long for tar format: "
2078 : : "file name \"%s\", target \"%s\"",
2079 : : filename, linktarget)));
2080 : : break;
2081 : 0 : default:
2082 [ # # ]: 0 : elog(ERROR, "unrecognized tar error: %d", rc);
2083 : : }
2084 : :
1642 rhaas@postgresql.org 2085 :CBC 194277 : bbsink_archive_contents(sink, TAR_BLOCK_SIZE);
2086 : : }
2087 : :
2088 : 199162 : return TAR_BLOCK_SIZE;
2089 : : }
2090 : :
2091 : : /*
2092 : : * Pad with zero bytes out to a multiple of TAR_BLOCK_SIZE.
2093 : : */
2094 : : static void
2095 : 189475 : _tarWritePadding(bbsink *sink, int len)
2096 : : {
2097 : 189475 : int pad = tarPaddingBytesRequired(len);
2098 : :
2099 : : /*
2100 : : * As in _tarWriteHeader, it should be safe to assume that the buffer is
2101 : : * large enough that we don't need to do this in multiple chunks.
2102 : : */
2103 [ - + ]: 189475 : Assert(sink->bbs_buffer_length >= TAR_BLOCK_SIZE);
2104 [ - + ]: 189475 : Assert(pad <= TAR_BLOCK_SIZE);
2105 : :
2106 [ + + ]: 189475 : if (pad > 0)
2107 : : {
2108 [ + - + + : 35929 : MemSet(sink->bbs_buffer, 0, pad);
+ - + - +
+ ]
2109 : 10518 : bbsink_archive_contents(sink, pad);
2110 : : }
4450 alvherre@alvh.no-ip. 2111 : 189475 : }
2112 : :
2113 : : /*
2114 : : * If the entry in statbuf is a link, then adjust statbuf to make it look like a
2115 : : * directory, so that it will be written that way.
2116 : : */
2117 : : static void
1642 rhaas@postgresql.org 2118 : 2900 : convert_link_to_directory(const char *pathbuf, struct stat *statbuf)
2119 : : {
2120 : : /* If symlink, write it as a directory anyway */
2121 [ + + ]: 2900 : if (S_ISLNK(statbuf->st_mode))
2122 : 63 : statbuf->st_mode = S_IFDIR | pg_dir_create_mode;
2254 fujii@postgresql.org 2123 : 2900 : }
2124 : :
2125 : : /*
2126 : : * Read some data from a file, setting a wait event and reporting any error
2127 : : * encountered.
2128 : : *
2129 : : * If partial_read_ok is false, also report an error if the number of bytes
2130 : : * read is not equal to the number of bytes requested.
2131 : : *
2132 : : * Returns the number of bytes read.
2133 : : */
2134 : : static ssize_t
2148 rhaas@postgresql.org 2135 : 228440 : basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset,
2136 : : const char *filename, bool partial_read_ok)
2137 : : {
2138 : : ssize_t rc;
2139 : :
2140 : 228440 : pgstat_report_wait_start(WAIT_EVENT_BASEBACKUP_READ);
1314 tmunro@postgresql.or 2141 : 228440 : rc = pg_pread(fd, buf, nbytes, offset);
2148 rhaas@postgresql.org 2142 : 228440 : pgstat_report_wait_end();
2143 : :
2144 [ - + ]: 228440 : if (rc < 0)
2148 rhaas@postgresql.org 2145 [ # # ]:UBC 0 : ereport(ERROR,
2146 : : (errcode_for_file_access(),
2147 : : errmsg("could not read file \"%s\": %m", filename)));
2148 rhaas@postgresql.org 2148 [ + + + - :CBC 228440 : if (!partial_read_ok && rc > 0 && rc != nbytes)
- + ]
2148 rhaas@postgresql.org 2149 [ # # ]:UBC 0 : ereport(ERROR,
2150 : : (errcode_for_file_access(),
2151 : : errmsg("could not read file \"%s\": read %zd of %zu",
2152 : : filename, rc, nbytes)));
2153 : :
2148 rhaas@postgresql.org 2154 :CBC 228440 : return rc;
2155 : : }
|
