Age Owner Branch data TLA Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * filemap.c
4 : : * A data structure for keeping track of files that have changed.
5 : : *
6 : : * This source file contains the logic to decide what to do with different
7 : : * kinds of files, and the data structure to support it. Before modifying
8 : : * anything, pg_rewind collects information about all the files and their
9 : : * attributes in the target and source data directories. It also scans the
10 : : * WAL log in the target, and collects information about data blocks that
11 : : * were changed. All this information is stored in a hash table, using the
12 : : * file path relative to the root of the data directory as the key.
13 : : *
14 : : * After collecting all the information required, the decide_file_actions()
15 : : * function scans the hash table and decides what action needs to be taken
16 : : * for each file. Finally, it sorts the array to the final order that the
17 : : * actions should be executed in.
18 : : *
19 : : * Copyright (c) 2013-2025, PostgreSQL Global Development Group
20 : : *
21 : : *-------------------------------------------------------------------------
22 : : */
23 : :
24 : : #include "postgres_fe.h"
25 : :
26 : : #include <sys/stat.h>
27 : : #include <unistd.h>
28 : :
29 : : #include "access/xlog_internal.h"
30 : : #include "catalog/pg_tablespace_d.h"
31 : : #include "common/file_utils.h"
32 : : #include "common/hashfn_unstable.h"
33 : : #include "common/string.h"
34 : : #include "datapagemap.h"
35 : : #include "filemap.h"
36 : : #include "pg_rewind.h"
37 : :
38 : : /*
39 : : * Define a hash table which we can use to store information about the files
40 : : * appearing in source and target systems.
41 : : */
42 : : #define SH_PREFIX filehash
43 : : #define SH_ELEMENT_TYPE file_entry_t
44 : : #define SH_KEY_TYPE const char *
45 : : #define SH_KEY path
46 : : #define SH_HASH_KEY(tb, key) hash_string(key)
47 : : #define SH_EQUAL(tb, a, b) (strcmp(a, b) == 0)
48 : : #define SH_SCOPE static inline
49 : : #define SH_RAW_ALLOCATOR pg_malloc0
50 : : #define SH_DECLARE
51 : : #define SH_DEFINE
52 : : #include "lib/simplehash.h"
53 : :
54 : : #define FILEHASH_INITIAL_SIZE 1000
55 : :
56 : : static filehash_hash *filehash;
57 : :
58 : : static bool isRelDataFile(const char *path);
59 : : static char *datasegpath(RelFileLocator rlocator, ForkNumber forknum,
60 : : BlockNumber segno);
61 : :
62 : : static file_entry_t *insert_filehash_entry(const char *path);
63 : : static file_entry_t *lookup_filehash_entry(const char *path);
64 : :
65 : : /*
66 : : * A separate hash table which tracks WAL files that must not be deleted.
67 : : */
68 : : typedef struct keepwal_entry
69 : : {
70 : : const char *path;
71 : : uint32 status;
72 : : } keepwal_entry;
73 : :
74 : : #define SH_PREFIX keepwal
75 : : #define SH_ELEMENT_TYPE keepwal_entry
76 : : #define SH_KEY_TYPE const char *
77 : : #define SH_KEY path
78 : : #define SH_HASH_KEY(tb, key) hash_string(key)
79 : : #define SH_EQUAL(tb, a, b) (strcmp(a, b) == 0)
80 : : #define SH_SCOPE static inline
81 : : #define SH_RAW_ALLOCATOR pg_malloc0
82 : : #define SH_DECLARE
83 : : #define SH_DEFINE
84 : : #include "lib/simplehash.h"
85 : :
86 : : #define KEEPWAL_INITIAL_SIZE 1000
87 : :
88 : :
89 : : static keepwal_hash *keepwal = NULL;
90 : : static bool keepwal_entry_exists(const char *path);
91 : :
92 : : static int final_filemap_cmp(const void *a, const void *b);
93 : :
94 : : static bool check_file_excluded(const char *path, bool is_source);
95 : :
96 : : /*
97 : : * Definition of one element part of an exclusion list, used to exclude
98 : : * contents when rewinding. "name" is the name of the file or path to
99 : : * check for exclusion. If "match_prefix" is true, any items matching
100 : : * the name as prefix are excluded.
101 : : */
102 : : struct exclude_list_item
103 : : {
104 : : const char *name;
105 : : bool match_prefix;
106 : : };
107 : :
108 : : /*
109 : : * The contents of these directories are removed or recreated during server
110 : : * start so they are not included in data processed by pg_rewind.
111 : : *
112 : : * Note: those lists should be kept in sync with what basebackup.c provides.
113 : : * Some of the values, contrary to what basebackup.c uses, are hardcoded as
114 : : * they are defined in backend-only headers. So this list is maintained
115 : : * with a best effort in mind.
116 : : */
117 : : static const char *const excludeDirContents[] =
118 : : {
119 : : /*
120 : : * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped
121 : : * because extensions like pg_stat_statements store data there.
122 : : */
123 : : "pg_stat_tmp", /* defined as PG_STAT_TMP_DIR */
124 : :
125 : : /*
126 : : * It is generally not useful to backup the contents of this directory
127 : : * even if the intention is to restore to another primary. See backup.sgml
128 : : * for a more detailed description.
129 : : */
130 : : "pg_replslot", /* defined as PG_REPLSLOT_DIR */
131 : :
132 : : /* Contents removed on startup, see dsm_cleanup_for_mmap(). */
133 : : "pg_dynshmem", /* defined as PG_DYNSHMEM_DIR */
134 : :
135 : : /* Contents removed on startup, see AsyncShmemInit(). */
136 : : "pg_notify",
137 : :
138 : : /*
139 : : * Old contents are loaded for possible debugging but are not required for
140 : : * normal operation, see SerialInit().
141 : : */
142 : : "pg_serial",
143 : :
144 : : /* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */
145 : : "pg_snapshots",
146 : :
147 : : /* Contents zeroed on startup, see StartupSUBTRANS(). */
148 : : "pg_subtrans",
149 : :
150 : : /* end of list */
151 : : NULL
152 : : };
153 : :
154 : : /*
155 : : * List of files excluded from filemap processing. Files are excluded
156 : : * if their prefix match.
157 : : */
158 : : static const struct exclude_list_item excludeFiles[] =
159 : : {
160 : : /* Skip auto conf temporary file. */
161 : : {"postgresql.auto.conf.tmp", false}, /* defined as PG_AUTOCONF_FILENAME */
162 : :
163 : : /* Skip current log file temporary file */
164 : : {"current_logfiles.tmp", false}, /* defined as
165 : : * LOG_METAINFO_DATAFILE_TMP */
166 : :
167 : : /* Skip relation cache because it is rebuilt on startup */
168 : : {"pg_internal.init", true}, /* defined as RELCACHE_INIT_FILENAME */
169 : :
170 : : /*
171 : : * If there is a backup_label or tablespace_map file, it indicates that a
172 : : * recovery failed and this cluster probably can't be rewound, but exclude
173 : : * them anyway if they are found.
174 : : */
175 : : {"backup_label", false}, /* defined as BACKUP_LABEL_FILE */
176 : : {"tablespace_map", false}, /* defined as TABLESPACE_MAP */
177 : :
178 : : /*
179 : : * If there's a backup_manifest, it belongs to a backup that was used to
180 : : * start this server. It is *not* correct for this backup. Our
181 : : * backup_manifest is injected into the backup separately if users want
182 : : * it.
183 : : */
184 : : {"backup_manifest", false},
185 : :
186 : : {"postmaster.pid", false},
187 : : {"postmaster.opts", false},
188 : :
189 : : /* end of list */
190 : : {NULL, false}
191 : : };
192 : :
193 : : /*
194 : : * Initialize the hash table for the file map.
195 : : */
196 : : void
1767 heikki.linnakangas@i 197 :CBC 14 : filehash_init(void)
198 : : {
199 : 14 : filehash = filehash_create(FILEHASH_INITIAL_SIZE, NULL);
3820 200 : 14 : }
201 : :
202 : : /* Look up entry for 'path', creating a new one if it doesn't exist */
203 : : static file_entry_t *
1767 204 : 31961 : insert_filehash_entry(const char *path)
205 : : {
206 : : file_entry_t *entry;
207 : : bool found;
208 : :
209 : 31961 : entry = filehash_insert(filehash, path, &found);
210 [ + + ]: 31961 : if (!found)
211 : : {
212 : 16676 : entry->path = pg_strdup(path);
213 : 16676 : entry->isrelfile = isRelDataFile(path);
214 : :
215 : 16676 : entry->target_exists = false;
216 : 16676 : entry->target_type = FILE_TYPE_UNDEFINED;
217 : 16676 : entry->target_size = 0;
218 : 16676 : entry->target_link_target = NULL;
219 : 16676 : entry->target_pages_to_overwrite.bitmap = NULL;
220 : 16676 : entry->target_pages_to_overwrite.bitmapsize = 0;
221 : :
222 : 16676 : entry->source_exists = false;
223 : 16676 : entry->source_type = FILE_TYPE_UNDEFINED;
224 : 16676 : entry->source_size = 0;
225 : 16676 : entry->source_link_target = NULL;
226 : :
227 : 16676 : entry->action = FILE_ACTION_UNDECIDED;
228 : : }
229 : :
230 : 31961 : return entry;
231 : : }
232 : :
233 : : static file_entry_t *
234 : 84625 : lookup_filehash_entry(const char *path)
235 : : {
236 : 84625 : return filehash_lookup(filehash, path);
237 : : }
238 : :
239 : : /*
240 : : * Initialize a hash table to store WAL file names that must be kept.
241 : : */
242 : : void
295 alvherre@alvh.no-ip. 243 : 14 : keepwal_init(void)
244 : : {
245 : : /* An initial hash size out of thin air */
246 : 14 : keepwal = keepwal_create(KEEPWAL_INITIAL_SIZE, NULL);
247 : 14 : }
248 : :
249 : : /* Mark the given file to prevent its removal */
250 : : void
251 : 19 : keepwal_add_entry(const char *path)
252 : : {
253 : : keepwal_entry *entry;
254 : : bool found;
255 : :
256 : : /* Should only be called with keepwal initialized */
257 [ - + ]: 19 : Assert(keepwal != NULL);
258 : :
259 : 19 : entry = keepwal_insert(keepwal, path, &found);
260 : :
261 [ + - ]: 19 : if (!found)
262 : 19 : entry->path = pg_strdup(path);
263 : 19 : }
264 : :
265 : : /* Return true if file is marked as not to be removed, false otherwise */
266 : : static bool
267 : 655 : keepwal_entry_exists(const char *path)
268 : : {
269 : 655 : return keepwal_lookup(keepwal, path) != NULL;
270 : : }
271 : :
272 : : /*
273 : : * Callback for processing source file list.
274 : : *
275 : : * This is called once for every file in the source server. We record the
276 : : * type and size of the file, so that decide_file_action() can later decide what
277 : : * to do with it.
278 : : */
279 : : void
1767 heikki.linnakangas@i 280 : 15997 : process_source_file(const char *path, file_type_t type, size_t size,
281 : : const char *link_target)
282 : : {
283 : : file_entry_t *entry;
284 : :
285 : : /*
286 : : * Pretend that pg_wal is a directory, even if it's really a symlink. We
287 : : * don't want to mess with the symlink itself, nor complain if it's a
288 : : * symlink in source but not in target or vice versa.
289 : : */
3243 rhaas@postgresql.org 290 [ + + - + ]: 15997 : if (strcmp(path, "pg_wal") == 0 && type == FILE_TYPE_SYMLINK)
3687 heikki.linnakangas@i 291 :UBC 0 : type = FILE_TYPE_DIRECTORY;
292 : :
293 : : /*
294 : : * sanity check: a filename that looks like a data file better be a
295 : : * regular file
296 : : */
3820 heikki.linnakangas@i 297 [ + + - + ]:CBC 15997 : if (type != FILE_TYPE_REGULAR && isRelDataFile(path))
2350 peter@eisentraut.org 298 :UBC 0 : pg_fatal("data file \"%s\" in source is not a regular file", path);
299 : :
300 : : /* Remember this source file */
1767 heikki.linnakangas@i 301 :CBC 15997 : entry = insert_filehash_entry(path);
302 [ - + ]: 15997 : if (entry->source_exists)
1767 heikki.linnakangas@i 303 :UBC 0 : pg_fatal("duplicate source file \"%s\"", path);
1767 heikki.linnakangas@i 304 :CBC 15997 : entry->source_exists = true;
305 : 15997 : entry->source_type = type;
306 : 15997 : entry->source_size = size;
307 [ + + ]: 15997 : entry->source_link_target = link_target ? pg_strdup(link_target) : NULL;
3820 308 : 15997 : }
309 : :
310 : : /*
311 : : * Callback for processing target file list.
312 : : *
313 : : * Record the type and size of the file, like process_source_file() does.
314 : : */
315 : : void
1767 316 : 15964 : process_target_file(const char *path, file_type_t type, size_t size,
317 : : const char *link_target)
318 : : {
319 : : file_entry_t *entry;
320 : :
321 : : /*
322 : : * Do not apply any exclusion filters here. This has advantage to remove
323 : : * from the target data folder all paths which have been filtered out from
324 : : * the source data folder when processing the source files.
325 : : */
326 : :
327 : : /*
328 : : * Like in process_source_file, pretend that pg_wal is always a directory.
329 : : */
3243 rhaas@postgresql.org 330 [ + + + + ]: 15964 : if (strcmp(path, "pg_wal") == 0 && type == FILE_TYPE_SYMLINK)
3687 heikki.linnakangas@i 331 : 2 : type = FILE_TYPE_DIRECTORY;
332 : :
333 : : /* Remember this target file */
1767 334 : 15964 : entry = insert_filehash_entry(path);
335 [ - + ]: 15964 : if (entry->target_exists)
1767 heikki.linnakangas@i 336 :UBC 0 : pg_fatal("duplicate source file \"%s\"", path);
1767 heikki.linnakangas@i 337 :CBC 15964 : entry->target_exists = true;
338 : 15964 : entry->target_type = type;
339 : 15964 : entry->target_size = size;
340 [ + + ]: 15964 : entry->target_link_target = link_target ? pg_strdup(link_target) : NULL;
3820 341 : 15964 : }
342 : :
343 : : /*
344 : : * This callback gets called while we read the WAL in the target, for every
345 : : * block that has changed in the target system. It decides if the given
346 : : * 'blkno' in the target relfile needs to be overwritten from the source, and
347 : : * if so, records it in 'target_pages_to_overwrite' bitmap.
348 : : *
349 : : * NOTE: All the files on both systems must have already been added to the
350 : : * hash table!
351 : : */
352 : : void
1158 rhaas@postgresql.org 353 : 84625 : process_target_wal_block_change(ForkNumber forknum, RelFileLocator rlocator,
354 : : BlockNumber blkno)
355 : : {
356 : : char *path;
357 : : file_entry_t *entry;
358 : : BlockNumber blkno_inseg;
359 : : int segno;
360 : :
3820 heikki.linnakangas@i 361 : 84625 : segno = blkno / RELSEG_SIZE;
362 : 84625 : blkno_inseg = blkno % RELSEG_SIZE;
363 : :
1158 rhaas@postgresql.org 364 : 84625 : path = datasegpath(rlocator, forknum, segno);
1767 heikki.linnakangas@i 365 : 84625 : entry = lookup_filehash_entry(path);
3740 fujii@postgresql.org 366 : 84625 : pfree(path);
367 : :
368 : : /*
369 : : * If the block still exists in both systems, remember it. Otherwise we
370 : : * can safely ignore it.
371 : : *
372 : : * If the block is beyond the EOF in the source system, or the file
373 : : * doesn't exist in the source at all, we're going to truncate/remove it
374 : : * away from the target anyway. Likewise, if it doesn't exist in the
375 : : * target anymore, we will copy it over with the "tail" from the source
376 : : * system, anyway.
377 : : *
378 : : * It is possible to find WAL for a file that doesn't exist on either
379 : : * system anymore. It means that the relation was dropped later in the
380 : : * target system, and independently on the source system too, or that it
381 : : * was created and dropped in the target system and it never existed in
382 : : * the source. Either way, we can safely ignore it.
383 : : */
3820 heikki.linnakangas@i 384 [ + - ]: 84625 : if (entry)
385 : : {
1767 386 [ - + ]: 84625 : Assert(entry->isrelfile);
387 : :
1761 388 [ + + ]: 84625 : if (entry->target_exists)
389 : : {
390 [ - + ]: 84621 : if (entry->target_type != FILE_TYPE_REGULAR)
1761 heikki.linnakangas@i 391 :UBC 0 : pg_fatal("unexpected page modification for non-regular file \"%s\"",
392 : : entry->path);
393 : :
1761 heikki.linnakangas@i 394 [ + + ]:CBC 84621 : if (entry->source_exists)
395 : : {
396 : : off_t end_offset;
397 : :
398 : 82925 : end_offset = (blkno_inseg + 1) * BLCKSZ;
399 [ + + + + ]: 82925 : if (end_offset <= entry->source_size && end_offset <= entry->target_size)
400 : 2925 : datapagemap_add(&entry->target_pages_to_overwrite, blkno_inseg);
401 : : }
402 : : }
403 : : }
3820 404 : 84625 : }
405 : :
406 : : /*
407 : : * Is this the path of file that pg_rewind can skip copying?
408 : : */
409 : : static bool
2550 peter_e@gmx.net 410 : 16660 : check_file_excluded(const char *path, bool is_source)
411 : : {
412 : : char localpath[MAXPGPATH];
413 : : int excludeIdx;
414 : : const char *filename;
415 : :
416 : : /*
417 : : * Skip all temporary files, .../pgsql_tmp/... and .../pgsql_tmp.*
418 : : */
1767 heikki.linnakangas@i 419 [ + + ]: 16660 : if (strstr(path, "/" PG_TEMP_FILE_PREFIX) != NULL ||
420 [ - + ]: 16646 : strstr(path, "/" PG_TEMP_FILES_DIR "/") != NULL)
421 : : {
422 : 14 : return true;
423 : : }
424 : :
425 : : /* check individual files... */
2021 michael@paquier.xyz 426 [ + + ]: 149530 : for (excludeIdx = 0; excludeFiles[excludeIdx].name != NULL; excludeIdx++)
427 : : {
428 : 132954 : int cmplen = strlen(excludeFiles[excludeIdx].name);
429 : :
2718 fujii@postgresql.org 430 : 132954 : filename = last_dir_separator(path);
431 [ + + ]: 132954 : if (filename == NULL)
432 : 2902 : filename = path;
433 : : else
434 : 130052 : filename++;
435 : :
2021 michael@paquier.xyz 436 [ + + ]: 132954 : if (!excludeFiles[excludeIdx].match_prefix)
437 : 116308 : cmplen++;
438 [ + + ]: 132954 : if (strncmp(filename, excludeFiles[excludeIdx].name, cmplen) == 0)
439 : : {
2550 peter_e@gmx.net 440 [ + - ]: 70 : if (is_source)
2350 peter@eisentraut.org 441 [ + - ]: 70 : pg_log_debug("entry \"%s\" excluded from source file list",
442 : : path);
443 : : else
2350 peter@eisentraut.org 444 [ # # ]:UBC 0 : pg_log_debug("entry \"%s\" excluded from target file list",
445 : : path);
2718 fujii@postgresql.org 446 :CBC 70 : return true;
447 : : }
448 : : }
449 : :
450 : : /*
451 : : * ... And check some directories. Note that this includes any contents
452 : : * within the directories themselves.
453 : : */
454 [ + + ]: 132594 : for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)
455 : : {
456 : 116032 : snprintf(localpath, sizeof(localpath), "%s/",
457 : 116032 : excludeDirContents[excludeIdx]);
458 [ + + ]: 116032 : if (strstr(path, localpath) == path)
459 : : {
2550 peter_e@gmx.net 460 [ + - ]: 14 : if (is_source)
2350 peter@eisentraut.org 461 [ + - ]: 14 : pg_log_debug("entry \"%s\" excluded from source file list",
462 : : path);
463 : : else
2350 peter@eisentraut.org 464 [ # # ]:UBC 0 : pg_log_debug("entry \"%s\" excluded from target file list",
465 : : path);
2718 fujii@postgresql.org 466 :CBC 14 : return true;
467 : : }
468 : : }
469 : :
470 : 16562 : return false;
471 : : }
472 : :
473 : : static const char *
3820 heikki.linnakangas@i 474 : 5591 : action_to_str(file_action_t action)
475 : : {
476 [ + + + + : 5591 : switch (action)
+ + - ]
477 : : {
478 : 415 : case FILE_ACTION_NONE:
479 : 415 : return "NONE";
480 : 4437 : case FILE_ACTION_COPY:
481 : 4437 : return "COPY";
482 : 4 : case FILE_ACTION_TRUNCATE:
483 : 4 : return "TRUNCATE";
484 : 5 : case FILE_ACTION_COPY_TAIL:
485 : 5 : return "COPY_TAIL";
486 : 9 : case FILE_ACTION_CREATE:
487 : 9 : return "CREATE";
488 : 721 : case FILE_ACTION_REMOVE:
489 : 721 : return "REMOVE";
490 : :
3820 heikki.linnakangas@i 491 :UBC 0 : default:
492 : 0 : return "unknown";
493 : : }
494 : : }
495 : :
496 : : /*
497 : : * Calculate the totals needed for progress reports.
498 : : */
499 : : void
1767 500 : 0 : calculate_totals(filemap_t *filemap)
501 : : {
502 : : file_entry_t *entry;
503 : : int i;
504 : :
505 : 0 : filemap->total_size = 0;
506 : 0 : filemap->fetch_size = 0;
507 : :
508 [ # # ]: 0 : for (i = 0; i < filemap->nentries; i++)
509 : : {
510 : 0 : entry = filemap->entries[i];
511 : :
512 [ # # ]: 0 : if (entry->source_type != FILE_TYPE_REGULAR)
3820 513 : 0 : continue;
514 : :
1767 515 : 0 : filemap->total_size += entry->source_size;
516 : :
3820 517 [ # # ]: 0 : if (entry->action == FILE_ACTION_COPY)
518 : : {
1767 519 : 0 : filemap->fetch_size += entry->source_size;
3820 520 : 0 : continue;
521 : : }
522 : :
523 [ # # ]: 0 : if (entry->action == FILE_ACTION_COPY_TAIL)
1767 524 : 0 : filemap->fetch_size += (entry->source_size - entry->target_size);
525 : :
526 [ # # ]: 0 : if (entry->target_pages_to_overwrite.bitmapsize > 0)
527 : : {
528 : : datapagemap_iterator_t *iter;
529 : : BlockNumber blk;
530 : :
531 : 0 : iter = datapagemap_iterate(&entry->target_pages_to_overwrite);
3820 532 [ # # ]: 0 : while (datapagemap_next(iter, &blk))
1767 533 : 0 : filemap->fetch_size += BLCKSZ;
534 : :
3820 535 : 0 : pg_free(iter);
536 : : }
537 : : }
538 : 0 : }
539 : :
540 : : void
1767 heikki.linnakangas@i 541 :CBC 14 : print_filemap(filemap_t *filemap)
542 : : {
543 : : file_entry_t *entry;
544 : : int i;
545 : :
546 [ + + ]: 16690 : for (i = 0; i < filemap->nentries; i++)
547 : : {
548 : 16676 : entry = filemap->entries[i];
3820 549 [ + + ]: 16676 : if (entry->action != FILE_ACTION_NONE ||
1767 550 [ + + ]: 11500 : entry->target_pages_to_overwrite.bitmapsize > 0)
551 : : {
2350 peter@eisentraut.org 552 [ + - ]: 5591 : pg_log_debug("%s (%s)", entry->path,
553 : : action_to_str(entry->action));
554 : :
1767 heikki.linnakangas@i 555 [ + + ]: 5591 : if (entry->target_pages_to_overwrite.bitmapsize > 0)
556 : 423 : datapagemap_print(&entry->target_pages_to_overwrite);
557 : : }
558 : : }
3820 559 : 14 : fflush(stdout);
560 : 14 : }
561 : :
562 : : /*
563 : : * Does it look like a relation data file?
564 : : *
565 : : * For our purposes, only files belonging to the main fork are considered
566 : : * relation files. Other forks are always copied in toto, because we cannot
567 : : * reliably track changes to them, because WAL only contains block references
568 : : * for the main fork.
569 : : */
570 : : static bool
571 : 17069 : isRelDataFile(const char *path)
572 : : {
573 : : RelFileLocator rlocator;
574 : : unsigned int segNo;
575 : : int nmatch;
576 : : bool matched;
577 : :
578 : : /*----
579 : : * Relation data files can be in one of the following directories:
580 : : *
581 : : * global/
582 : : * shared relations
583 : : *
584 : : * base/<db oid>/
585 : : * regular relations, default tablespace
586 : : *
587 : : * pg_tblspc/<tblspc oid>/<tblspc version>/
588 : : * within a non-default tablespace (the name of the directory
589 : : * depends on version)
590 : : *
591 : : * And the relation data files themselves have a filename like:
592 : : *
593 : : * <oid>.<segment number>
594 : : *
595 : : *----
596 : : */
1158 rhaas@postgresql.org 597 : 17069 : rlocator.spcOid = InvalidOid;
598 : 17069 : rlocator.dbOid = InvalidOid;
599 : 17069 : rlocator.relNumber = InvalidRelFileNumber;
3820 heikki.linnakangas@i 600 : 17069 : segNo = 0;
601 : 17069 : matched = false;
602 : :
1074 rhaas@postgresql.org 603 : 17069 : nmatch = sscanf(path, "global/%u.%u", &rlocator.relNumber, &segNo);
3820 heikki.linnakangas@i 604 [ + + - + ]: 17069 : if (nmatch == 1 || nmatch == 2)
605 : : {
1158 rhaas@postgresql.org 606 : 784 : rlocator.spcOid = GLOBALTABLESPACE_OID;
607 : 784 : rlocator.dbOid = 0;
3820 heikki.linnakangas@i 608 : 784 : matched = true;
609 : : }
610 : : else
611 : : {
1074 rhaas@postgresql.org 612 : 16285 : nmatch = sscanf(path, "base/%u/%u.%u",
613 : : &rlocator.dbOid, &rlocator.relNumber, &segNo);
3820 heikki.linnakangas@i 614 [ + + - + ]: 16285 : if (nmatch == 2 || nmatch == 3)
615 : : {
1158 rhaas@postgresql.org 616 : 14987 : rlocator.spcOid = DEFAULTTABLESPACE_OID;
3820 heikki.linnakangas@i 617 : 14987 : matched = true;
618 : : }
619 : : else
620 : : {
1074 rhaas@postgresql.org 621 : 1298 : nmatch = sscanf(path, "pg_tblspc/%u/" TABLESPACE_VERSION_DIRECTORY "/%u/%u.%u",
622 : : &rlocator.spcOid, &rlocator.dbOid, &rlocator.relNumber,
623 : : &segNo);
2741 fujii@postgresql.org 624 [ + + - + ]: 1298 : if (nmatch == 3 || nmatch == 4)
3820 heikki.linnakangas@i 625 : 12 : matched = true;
626 : : }
627 : : }
628 : :
629 : : /*
630 : : * The sscanf tests above can match files that have extra characters at
631 : : * the end. To eliminate such cases, cross-check that GetRelationPath
632 : : * creates the exact same filename, when passed the RelFileLocator
633 : : * information we extracted from the filename.
634 : : */
635 [ + + ]: 17069 : if (matched)
636 : : {
1158 rhaas@postgresql.org 637 : 15783 : char *check_path = datasegpath(rlocator, MAIN_FORKNUM, segNo);
638 : :
3820 heikki.linnakangas@i 639 [ + + ]: 15783 : if (strcmp(check_path, path) != 0)
640 : 3852 : matched = false;
641 : :
642 : 15783 : pfree(check_path);
643 : : }
644 : :
645 : 17069 : return matched;
646 : : }
647 : :
648 : : /*
649 : : * A helper function to create the path of a relation file and segment.
650 : : *
651 : : * The returned path is palloc'd
652 : : */
653 : : static char *
1158 rhaas@postgresql.org 654 : 100408 : datasegpath(RelFileLocator rlocator, ForkNumber forknum, BlockNumber segno)
655 : : {
656 : : RelPathStr path;
657 : : char *segpath;
658 : :
659 : 100408 : path = relpathperm(rlocator, forknum);
3820 heikki.linnakangas@i 660 [ - + ]: 100408 : if (segno > 0)
661 : : {
193 andres@anarazel.de 662 :UBC 0 : segpath = psprintf("%s.%u", path.str, segno);
3820 heikki.linnakangas@i 663 : 0 : return segpath;
664 : : }
665 : : else
193 andres@anarazel.de 666 :CBC 100408 : return pstrdup(path.str);
667 : : }
668 : :
669 : : /*
670 : : * In the final stage, the filemap is sorted so that removals come last.
671 : : * From disk space usage point of view, it would be better to do removals
672 : : * first, but for now, safety first. If a whole directory is deleted, all
673 : : * files and subdirectories inside it need to removed first. On creation,
674 : : * parent directory needs to be created before files and directories inside
675 : : * it. To achieve that, the file_action_t enum is ordered so that we can
676 : : * just sort on that first. Furthermore, sort REMOVE entries in reverse
677 : : * path order, so that "foo/bar" subdirectory is removed before "foo".
678 : : */
679 : : static int
3820 heikki.linnakangas@i 680 : 178339 : final_filemap_cmp(const void *a, const void *b)
681 : : {
682 : 178339 : file_entry_t *fa = *((file_entry_t **) a);
683 : 178339 : file_entry_t *fb = *((file_entry_t **) b);
684 : :
685 [ + + ]: 178339 : if (fa->action > fb->action)
686 : 6618 : return 1;
687 [ + + ]: 171721 : if (fa->action < fb->action)
688 : 9561 : return -1;
689 : :
690 [ + + ]: 162160 : if (fa->action == FILE_ACTION_REMOVE)
2528 tgl@sss.pgh.pa.us 691 : 5670 : return strcmp(fb->path, fa->path);
692 : : else
3820 heikki.linnakangas@i 693 : 156490 : return strcmp(fa->path, fb->path);
694 : : }
695 : :
696 : : /*
697 : : * Decide what action to perform to a file.
698 : : */
699 : : static file_action_t
1767 700 : 16676 : decide_file_action(file_entry_t *entry)
701 : : {
702 : 16676 : const char *path = entry->path;
703 : :
704 : : /*
705 : : * Don't touch the control file. It is handled specially, after copying
706 : : * all the other files.
707 : : */
152 fujii@postgresql.org 708 [ + + ]: 16676 : if (strcmp(path, XLOG_CONTROL_FILE) == 0)
1767 heikki.linnakangas@i 709 : 14 : return FILE_ACTION_NONE;
710 : :
711 : : /* Skip macOS system files */
571 dgustafsson@postgres 712 [ + + ]: 16662 : if (strstr(path, ".DS_Store") != NULL)
713 : 2 : return FILE_ACTION_NONE;
714 : :
715 : : /*
716 : : * Remove all files matching the exclusion filters in the target.
717 : : */
1767 heikki.linnakangas@i 718 [ + + ]: 16660 : if (check_file_excluded(path, true))
719 : : {
720 [ + + ]: 98 : if (entry->target_exists)
721 : 70 : return FILE_ACTION_REMOVE;
722 : : else
723 : 28 : return FILE_ACTION_NONE;
724 : : }
725 : :
726 : : /*
727 : : * Handle cases where the file is missing from one of the systems.
728 : : */
729 [ + + + - ]: 16562 : if (!entry->target_exists && entry->source_exists)
730 : : {
731 : : /*
732 : : * File exists in source, but not in target. Copy it in toto. (If it's
733 : : * a relation data file, WAL replay after rewinding should re-create
734 : : * it anyway. But there's no harm in copying it now.)
735 : : */
736 [ + + - - ]: 682 : switch (entry->source_type)
737 : : {
738 : 9 : case FILE_TYPE_DIRECTORY:
739 : : case FILE_TYPE_SYMLINK:
740 : 9 : return FILE_ACTION_CREATE;
741 : 673 : case FILE_TYPE_REGULAR:
742 : 673 : return FILE_ACTION_COPY;
1767 heikki.linnakangas@i 743 :UBC 0 : case FILE_TYPE_UNDEFINED:
744 : 0 : pg_fatal("unknown file type for \"%s\"", entry->path);
745 : : break;
746 : : }
747 : : }
1767 heikki.linnakangas@i 748 [ + - + + ]:CBC 15880 : else if (entry->target_exists && !entry->source_exists)
749 : : {
750 : : /*
751 : : * For files that exist in target but not in source, we check the
752 : : * keepwal hash table; any files listed therein must not be removed.
753 : : */
295 alvherre@alvh.no-ip. 754 [ + + ]: 655 : if (keepwal_entry_exists(path))
755 : : {
756 [ + - ]: 4 : pg_log_debug("Not removing file \"%s\" because it is required for recovery", path);
757 : 4 : return FILE_ACTION_NONE;
758 : : }
1767 heikki.linnakangas@i 759 : 651 : return FILE_ACTION_REMOVE;
760 : : }
761 [ - + - - ]: 15225 : else if (!entry->target_exists && !entry->source_exists)
762 : : {
763 : : /*
764 : : * Doesn't exist in either server. Why does it have an entry in the
765 : : * first place??
766 : : */
1767 heikki.linnakangas@i 767 :UBC 0 : Assert(false);
768 : : return FILE_ACTION_NONE;
769 : : }
770 : :
771 : : /*
772 : : * Otherwise, the file exists on both systems
773 : : */
1767 heikki.linnakangas@i 774 [ + - + - ]:CBC 15225 : Assert(entry->target_exists && entry->source_exists);
775 : :
776 [ - + ]: 15225 : if (entry->source_type != entry->target_type)
777 : : {
778 : : /* But it's a different kind of object. Strange.. */
1767 heikki.linnakangas@i 779 :UBC 0 : pg_fatal("file \"%s\" is of different type in source and target", entry->path);
780 : : }
781 : :
782 : : /*
783 : : * PG_VERSION files should be identical on both systems, but avoid
784 : : * overwriting them for paranoia.
785 : : */
1767 heikki.linnakangas@i 786 [ + + ]:CBC 15225 : if (pg_str_endswith(entry->path, "PG_VERSION"))
787 : 60 : return FILE_ACTION_NONE;
788 : :
789 [ + - + - : 15165 : switch (entry->source_type)
- ]
790 : : {
791 : 382 : case FILE_TYPE_DIRECTORY:
792 : 382 : return FILE_ACTION_NONE;
793 : :
1767 heikki.linnakangas@i 794 :UBC 0 : case FILE_TYPE_SYMLINK:
795 : :
796 : : /*
797 : : * XXX: Should we check if it points to the same target?
798 : : */
799 : 0 : return FILE_ACTION_NONE;
800 : :
1767 heikki.linnakangas@i 801 :CBC 14783 : case FILE_TYPE_REGULAR:
802 [ + + ]: 14783 : if (!entry->isrelfile)
803 : : {
804 : : /*
805 : : * It's a non-data file that we have no special processing
806 : : * for. Copy it in toto.
807 : : */
808 : 3764 : return FILE_ACTION_COPY;
809 : : }
810 : : else
811 : : {
812 : : /*
813 : : * It's a data file that exists in both systems.
814 : : *
815 : : * If it's larger in target, we can truncate it. There will
816 : : * also be a WAL record of the truncation in the source
817 : : * system, so WAL replay would eventually truncate the target
818 : : * too, but we might as well do it now.
819 : : *
820 : : * If it's smaller in the target, it means that it has been
821 : : * truncated in the target, or enlarged in the source, or
822 : : * both. If it was truncated in the target, we need to copy
823 : : * the missing tail from the source system. If it was enlarged
824 : : * in the source system, there will be WAL records in the
825 : : * source system for the new blocks, so we wouldn't need to
826 : : * copy them here. But we don't know which scenario we're
827 : : * dealing with, and there's no harm in copying the missing
828 : : * blocks now, so do it now.
829 : : *
830 : : * If it's the same size, do nothing here. Any blocks modified
831 : : * in the target will be copied based on parsing the target
832 : : * system's WAL, and any blocks modified in the source will be
833 : : * updated after rewinding, when the source system's WAL is
834 : : * replayed.
835 : : */
836 [ + + ]: 11019 : if (entry->target_size < entry->source_size)
837 : 5 : return FILE_ACTION_COPY_TAIL;
838 [ + + ]: 11014 : else if (entry->target_size > entry->source_size)
839 : 4 : return FILE_ACTION_TRUNCATE;
840 : : else
841 : 11010 : return FILE_ACTION_NONE;
842 : : }
843 : : break;
844 : :
1767 heikki.linnakangas@i 845 :UBC 0 : case FILE_TYPE_UNDEFINED:
846 : 0 : pg_fatal("unknown file type for \"%s\"", path);
847 : : break;
848 : : }
849 : :
850 : : /* unreachable */
851 : 0 : pg_fatal("could not decide what to do with file \"%s\"", path);
852 : : }
853 : :
854 : : /*
855 : : * Decide what to do with each file.
856 : : *
857 : : * Returns a 'filemap' with the entries in the order that their actions
858 : : * should be executed.
859 : : */
860 : : filemap_t *
1767 heikki.linnakangas@i 861 :CBC 14 : decide_file_actions(void)
862 : : {
863 : : int i;
864 : : filehash_iterator it;
865 : : file_entry_t *entry;
866 : : filemap_t *filemap;
867 : :
868 : 14 : filehash_start_iterate(filehash, &it);
869 [ + + ]: 16690 : while ((entry = filehash_iterate(filehash, &it)) != NULL)
870 : : {
871 : 16676 : entry->action = decide_file_action(entry);
872 : : }
873 : :
874 : : /*
875 : : * Turn the hash table into an array, and sort in the order that the
876 : : * actions should be performed.
877 : : */
878 : 14 : filemap = pg_malloc(offsetof(filemap_t, entries) +
879 : 14 : filehash->members * sizeof(file_entry_t *));
880 : 14 : filemap->nentries = filehash->members;
881 : 14 : filehash_start_iterate(filehash, &it);
882 : 14 : i = 0;
883 [ + + ]: 16690 : while ((entry = filehash_iterate(filehash, &it)) != NULL)
884 : : {
885 : 16676 : filemap->entries[i++] = entry;
886 : : }
887 : :
888 : 14 : qsort(&filemap->entries, filemap->nentries, sizeof(file_entry_t *),
889 : : final_filemap_cmp);
890 : :
891 : 14 : return filemap;
892 : : }
|
