From e8f69be054e9343b3c41d7e77cc142913ee55439 Mon Sep 17 00:00:00 2001 From: Philip Warner Date: Fri, 21 Jul 2000 11:40:08 +0000 Subject: [PATCH] - Support for BLOB output from pg_dump and input via pg_restore - Support for direct DB connection in pg_restore - Fixes in support for --insert flag - pg_dump now outputs in modified OID order - various other bug fixes --- src/bin/pg_dump/Makefile | 6 +- src/bin/pg_dump/README | 44 +- src/bin/pg_dump/pg_backup.h | 279 ++++--- src/bin/pg_dump/pg_backup_archiver.c | 872 +++++++++++++------ src/bin/pg_dump/pg_backup_archiver.h | 122 ++- src/bin/pg_dump/pg_backup_custom.c | 1519 +++++++++++++++++++++------------- src/bin/pg_dump/pg_backup_files.c | 786 +++++++++++------- src/bin/pg_dump/pg_dump.c | 374 +++++---- src/bin/pg_dump/pg_restore.c | 696 ++++++++-------- 9 files changed, 2924 insertions(+), 1774 deletions(-) diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile index 4f03eb3f25..6b10402861 100644 --- a/src/bin/pg_dump/Makefile +++ b/src/bin/pg_dump/Makefile @@ -4,7 +4,7 @@ # # Copyright (c) 1994, Regents of the University of California # -# $Header: /cvsroot/pgsql/src/bin/pg_dump/Makefile,v 1.19 2000/07/04 19:52:00 petere Exp $ +# $Header: /cvsroot/pgsql/src/bin/pg_dump/Makefile,v 1.20 2000/07/21 11:40:08 pjw Exp $ # #------------------------------------------------------------------------- @@ -12,8 +12,8 @@ subdir = src/bin/pg_dump top_builddir = ../../.. include ../../Makefile.global -OBJS= pg_backup_archiver.o pg_backup_custom.o pg_backup_files.o \ - pg_backup_plain_text.o $(STRDUP) +OBJS= pg_backup_archiver.o pg_backup_db.o pg_backup_custom.o pg_backup_files.o \ + pg_backup_null.o pg_backup_tar.o $(STRDUP) CFLAGS+= -I$(LIBPQDIR) LIBS+= -lz diff --git a/src/bin/pg_dump/README b/src/bin/pg_dump/README index 0dfa0aabe8..a386ac5a01 100644 --- a/src/bin/pg_dump/README +++ b/src/bin/pg_dump/README @@ -1,17 +1,23 @@ Notes on pg_dump ================ -pg_dump, by default, still outputs text files. +1. pg_dump, by default, still outputs text files. -pg_dumpall forces all pg_dump output to be text, since it also outputs text into the same output stream. +2. pg_dumpall forces all pg_dump output to be text, since it also outputs text into the same output stream. -The plain text output format can not be used as input into pg_restore. +3. The plain text output format can not be used as input into pg_restore. + +4. pg_dump now dumps the items in a modified OID order to try to improve relaibility of default restores. To dump a database into the next custom format, type: pg_dump -Fc > +or, in TAR format + + pg_dump -Ft > + To restore, try To list contents: @@ -53,7 +59,37 @@ or, simply: pg_restore backup.bck --use=toc.lis | psql newdbname -Philip Warner, 3-Jul-2000 +BLOBs +===== + +To dump blobs you must use the custom archive format (-Fc) or TAR format (-Ft), and specify the +--blobs qualifier to the pg_dump command. + +To restore blobs you must use a direct database connection (--db=db-to-restore-to). + +eg. + + pg_dump --blob -Fc db-to-backup -f backup.bck + + pg_restore backup.bck --db=db-to-restore-into + + +TAR +=== + +The TAR archive that pg_dump creates currently has a blank username & group for the files, +but should be otherwise valid. It also includes a 'restore.sql' script which is there for +the benefit of humans. It is never used by pg_restore. + +Note: the TAR format archive can only be used as input into pg_restore if it is in TAR form. +(ie. you should not extract the files then expect pg_restore to work). + +You can extract, edit, and tar the files again, and it should work, but the 'toc' +file should go at the start, the data files be in the order they are used, and +the BLOB files at the end. + + +Philip Warner, 16-Jul-2000 pjw@rhyme.com.au diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index e84b4dd16c..fb930637cc 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -1,125 +1,154 @@ -/*------------------------------------------------------------------------- - * - * pg_backup.h - * - * Public interface to the pg_dump archiver routines. - * - * See the headers to pg_restore for more details. - * - * Copyright (c) 2000, Philip Warner - * Rights are granted to use this software in any way so long - * as this notice is not removed. - * - * The author is not responsible for loss or damages that may - * result from it's use. - * - * - * IDENTIFICATION - * - * Modifications - 28-Jun-2000 - pjw@rhyme.com.au - * - * Initial version. - * - *------------------------------------------------------------------------- - */ - -#ifndef PG_BACKUP__ - -#include "config.h" -#include "c.h" - -#define PG_BACKUP__ - -typedef enum _archiveFormat { - archUnknown = 0, - archCustom = 1, - archFiles = 2, - archTar = 3, - archPlainText = 4 -} ArchiveFormat; - -/* - * We may want to have so user-readbale data, but in the mean - * time this gives us some abstraction and type checking. - */ -typedef struct _Archive { - /* Nothing here */ -} Archive; - -typedef int (*DataDumperPtr)(Archive* AH, char* oid, void* userArg); - -typedef struct _restoreOptions { - int dataOnly; - int dropSchema; - char *filename; - int schemaOnly; - int verbose; - int aclsSkip; - int tocSummary; - char *tocFile; - int oidOrder; - int origOrder; - int rearrange; - int format; - char *formatName; - - int selTypes; - int selIndex; - int selFunction; - int selTrigger; - int selTable; - char *indexNames; - char *functionNames; - char *tableNames; - char *triggerNames; - - int *idWanted; - int limitToList; - int compression; - -} RestoreOptions; - -/* - * Main archiver interface. - */ - -/* Called to add a TOC entry */ -extern void ArchiveEntry(Archive* AH, const char* oid, const char* name, - const char* desc, const char* (deps[]), const char* defn, - const char* dropStmt, const char* owner, - DataDumperPtr dumpFn, void* dumpArg); - -/* Called to write *data* to the archive */ -extern int WriteData(Archive* AH, const void* data, int dLen); - -extern void CloseArchive(Archive* AH); - -extern void RestoreArchive(Archive* AH, RestoreOptions *ropt); - -/* Open an existing archive */ -extern Archive* OpenArchive(const char* FileSpec, ArchiveFormat fmt); - -/* Create a new archive */ -extern Archive* CreateArchive(const char* FileSpec, ArchiveFormat fmt, int compression); - -/* The --list option */ -extern void PrintTOCSummary(Archive* AH, RestoreOptions *ropt); - -extern RestoreOptions* NewRestoreOptions(void); - -/* Rearrange TOC entries */ -extern void MoveToStart(Archive* AH, char *oType); -extern void MoveToEnd(Archive* AH, char *oType); -extern void SortTocByOID(Archive* AH); -extern void SortTocByID(Archive* AH); -extern void SortTocFromFile(Archive* AH, RestoreOptions *ropt); - -/* Convenience functions used only when writing DATA */ -extern int archputs(const char *s, Archive* AH); -extern int archputc(const char c, Archive* AH); -extern int archprintf(Archive* AH, const char *fmt, ...); - -#endif - - - +/*------------------------------------------------------------------------- + * + * pg_backup.h + * + * Public interface to the pg_dump archiver routines. + * + * See the headers to pg_restore for more details. + * + * Copyright (c) 2000, Philip Warner + * Rights are granted to use this software in any way so long + * as this notice is not removed. + * + * The author is not responsible for loss or damages that may + * result from it's use. + * + * + * IDENTIFICATION + * + * Modifications - 28-Jun-2000 - pjw@rhyme.com.au + * + * Initial version. + * + *------------------------------------------------------------------------- + */ + +#ifndef PG_BACKUP__ + +#include "config.h" +#include "c.h" + +#define PG_BACKUP__ + +#include "postgres.h" +#include "libpq-fe.h" + +typedef enum _archiveFormat { + archUnknown = 0, + archCustom = 1, + archFiles = 2, + archTar = 3, + archNull = 4 +} ArchiveFormat; + +/* + * We may want to have so user-readbale data, but in the mean + * time this gives us some abstraction and type checking. + */ +typedef struct _Archive { + int verbose; + /* The rest is private */ +} Archive; + +typedef int (*DataDumperPtr)(Archive* AH, char* oid, void* userArg); + +typedef struct _restoreOptions { + int dataOnly; + int dropSchema; + char *filename; + int schemaOnly; + int verbose; + int aclsSkip; + int tocSummary; + char *tocFile; + int oidOrder; + int origOrder; + int rearrange; + int format; + char *formatName; + + int selTypes; + int selIndex; + int selFunction; + int selTrigger; + int selTable; + char *indexNames; + char *functionNames; + char *tableNames; + char *triggerNames; + + int useDB; + char *dbname; + char *pgport; + char *pghost; + int ignoreVersion; + int requirePassword; + + int *idWanted; + int limitToList; + int compression; + +} RestoreOptions; + +/* + * Main archiver interface. + */ + +extern void exit_horribly(Archive *AH, const char *fmt, ...); + +/* Lets the archibe know we have a DB connection to shutdown if it dies */ + +PGconn* ConnectDatabase(Archive *AH, + const char* dbname, + const char* pghost, + const char* pgport, + const int reqPwd, + const int ignoreVersion); + + +/* Called to add a TOC entry */ +extern void ArchiveEntry(Archive* AH, const char* oid, const char* name, + const char* desc, const char* (deps[]), const char* defn, + const char* dropStmt, const char* copyStmt, const char* owner, + DataDumperPtr dumpFn, void* dumpArg); + +/* Called to write *data* to the archive */ +extern int WriteData(Archive* AH, const void* data, int dLen); + +//extern int StartBlobs(Archive* AH); +//extern int EndBlobs(Archive* AH); +extern int StartBlob(Archive* AH, int oid); +extern int EndBlob(Archive* AH, int oid); + +extern void CloseArchive(Archive* AH); + +extern void RestoreArchive(Archive* AH, RestoreOptions *ropt); + +/* Open an existing archive */ +extern Archive* OpenArchive(const char* FileSpec, const ArchiveFormat fmt); + +/* Create a new archive */ +extern Archive* CreateArchive(const char* FileSpec, const ArchiveFormat fmt, + const int compression); + +/* The --list option */ +extern void PrintTOCSummary(Archive* AH, RestoreOptions *ropt); + +extern RestoreOptions* NewRestoreOptions(void); + +/* Rearrange TOC entries */ +extern void MoveToStart(Archive* AH, char *oType); +extern void MoveToEnd(Archive* AH, char *oType); +extern void SortTocByOID(Archive* AH); +extern void SortTocByID(Archive* AH); +extern void SortTocFromFile(Archive* AH, RestoreOptions *ropt); + +/* Convenience functions used only when writing DATA */ +extern int archputs(const char *s, Archive* AH); +extern int archputc(const char c, Archive* AH); +extern int archprintf(Archive* AH, const char *fmt, ...); + +#endif + + + diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index e25cf56756..e210780196 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -25,6 +25,8 @@ #include "pg_backup.h" #include "pg_backup_archiver.h" +#include "pg_backup_db.h" + #include #include /* for dup */ @@ -32,10 +34,13 @@ #include #include +#include "pqexpbuffer.h" +#include "libpq/libpq-fs.h" + static void _SortToc(ArchiveHandle* AH, TocSortCompareFn fn); static int _tocSortCompareByOIDNum(const void *p1, const void *p2); static int _tocSortCompareByIDNum(const void *p1, const void *p2); -static ArchiveHandle* _allocAH(const char* FileSpec, ArchiveFormat fmt, +static ArchiveHandle* _allocAH(const char* FileSpec, const ArchiveFormat fmt, int compression, ArchiveMode mode); static int _printTocEntry(ArchiveHandle* AH, TocEntry* te, RestoreOptions *ropt); static int _tocEntryRequired(TocEntry* te, RestoreOptions *ropt); @@ -45,8 +50,14 @@ static TocEntry* _getTocEntry(ArchiveHandle* AH, int id); static void _moveAfter(ArchiveHandle* AH, TocEntry* pos, TocEntry* te); static void _moveBefore(ArchiveHandle* AH, TocEntry* pos, TocEntry* te); static int _discoverArchiveFormat(ArchiveHandle* AH); + static char *progname = "Archiver"; +static void _die_horribly(ArchiveHandle *AH, const char *fmt, va_list ap); + +static int _canRestoreBlobs(ArchiveHandle *AH); + + /* * Wrapper functions. * @@ -58,7 +69,9 @@ static char *progname = "Archiver"; /* Create a new archive */ /* Public */ -Archive* CreateArchive(const char* FileSpec, ArchiveFormat fmt, int compression) +Archive* CreateArchive(const char* FileSpec, const ArchiveFormat fmt, + const int compression) + { ArchiveHandle* AH = _allocAH(FileSpec, fmt, compression, archModeWrite); return (Archive*)AH; @@ -66,7 +79,7 @@ Archive* CreateArchive(const char* FileSpec, ArchiveFormat fmt, int compression) /* Open an existing archive */ /* Public */ -Archive* OpenArchive(const char* FileSpec, ArchiveFormat fmt) +Archive* OpenArchive(const char* FileSpec, const ArchiveFormat fmt) { ArchiveHandle* AH = _allocAH(FileSpec, fmt, 0, archModeRead); return (Archive*)AH; @@ -80,9 +93,9 @@ void CloseArchive(Archive* AHX) /* Close the output */ if (AH->gzOut) - GZCLOSE(AH->OF); + GZCLOSE(AH->OF); else if (AH->OF != stdout) - fclose(AH->OF); + fclose(AH->OF); } /* Public */ @@ -93,47 +106,155 @@ void RestoreArchive(Archive* AHX, RestoreOptions *ropt) int reqs; OutputContext sav; + AH->ropt = ropt; + + /* + * If we're using a DB connection, then connect it. + */ + if (ropt->useDB) + { + ahlog(AH, 1, "Connecting to database for restore\n"); + if (AH->version < K_VERS_1_3) + die_horribly(AH, "Direct database connections are not supported in pre-1.3 archives"); + + ConnectDatabase(AHX, ropt->dbname, ropt->pghost, ropt->pgport, + ropt->requirePassword, ropt->ignoreVersion); + } + + /* + * Setup the output file if necessary. + */ if (ropt->filename || ropt->compression) - sav = SetOutput(AH, ropt->filename, ropt->compression); + sav = SetOutput(AH, ropt->filename, ropt->compression); ahprintf(AH, "--\n-- Selected TOC Entries:\n--\n"); - /* Drop the items at the start, in reverse order */ + /* + * Drop the items at the start, in reverse order + */ if (ropt->dropSchema) { - te = AH->toc->prev; - while (te != AH->toc) { - reqs = _tocEntryRequired(te, ropt); - if ( (reqs & 1) && te->dropStmt) { /* We want the schema */ - ahprintf(AH, "%s", te->dropStmt); - } - te = te->prev; - } + te = AH->toc->prev; + while (te != AH->toc) { + reqs = _tocEntryRequired(te, ropt); + if ( ( (reqs & 1) != 0) && te->dropStmt) { /* We want the schema */ + ahlog(AH, 1, "Dropping %s %s\n", te->desc, te->name); + ahprintf(AH, "%s", te->dropStmt); + } + te = te->prev; + } } + /* + * Now process each TOC entry + */ te = AH->toc->next; while (te != AH->toc) { - reqs = _tocEntryRequired(te, ropt); - if (reqs & 1) /* We want the schema */ - _printTocEntry(AH, te, ropt); + /* Work out what, if anything, we want from this entry */ + reqs = _tocEntryRequired(te, ropt); + + if ( (reqs & 1) != 0) /* We want the schema */ + { + ahlog(AH, 1, "Creating %s %s\n", te->desc, te->name); + _printTocEntry(AH, te, ropt); + } - if (AH->PrintTocDataPtr != NULL && (reqs & 2) != 0) { + /* + * If we want data, and it has data, then restore that too + */ + if (AH->PrintTocDataPtr != NULL && (reqs & 2) != 0) { #ifndef HAVE_LIBZ - if (AH->compression != 0) - die_horribly("%s: Unable to restore data from a compressed archive\n", progname); + if (AH->compression != 0) + die_horribly(AH, "%s: Unable to restore data from a compressed archive\n", progname); #endif - _disableTriggers(AH, te, ropt); - (*AH->PrintTocDataPtr)(AH, te, ropt); - _enableTriggers(AH, te, ropt); - } - te = te->next; + + ahlog(AH, 1, "Restoring data for %s \n", te->name); + + ahprintf(AH, "--\n-- Data for TOC Entry ID %d (OID %s) %s %s\n--\n\n", + te->id, te->oid, te->desc, te->name); + + /* + * Maybe we can't do BLOBS, so check if this node is for BLOBS + */ + if ((strcmp(te->desc,"BLOBS") == 0) && !_canRestoreBlobs(AH)) + { + ahprintf(AH, "--\n-- SKIPPED \n--\n\n"); + /* + * This is a bit nasty - we assume, for the moment, that if a custom + * output is used, then we don't want warnings. + */ + if (!AH->CustomOutPtr) + fprintf(stderr, "%s: WARNING - skipping BLOB restoration\n", progname); + } else { + + _disableTriggers(AH, te, ropt); + + + /* If we have a copy statement, use it. As of V1.3, these are separate + * to allow easy import from withing a database connection. Pre 1.3 + * archives can not use DB connections and are sent to output only. + * + * For V1.3+, the table data MUST have a copy statement so that + * we can go into appropriate mode with libpq. + */ + if (te->copyStmt && strlen(te->copyStmt) > 0) + ahprintf(AH, te->copyStmt); + + (*AH->PrintTocDataPtr)(AH, te, ropt); + + _enableTriggers(AH, te, ropt); + } + } + te = te->next; } + /* + * Now use blobs_xref (if used) to fixup any refs for tables that we loaded + */ + if (_canRestoreBlobs(AH) && AH->createdBlobXref) + { + te = AH->toc->next; + while (te != AH->toc) { + + /* Is it table data? */ + if (strcmp(te->desc, "TABLE DATA") == 0) { + + ahlog(AH, 2, "Checking if we loaded %s\n", te->name); + + reqs = _tocEntryRequired(te, ropt); + + if ( (reqs & 2) != 0) /* We loaded the data */ + { + ahlog(AH, 1, "Fixing up BLOB ref for %s\n", te->name); + FixupBlobRefs(AH, te->name); + } + } + else + { + ahlog(AH, 2, "Ignoring BLOB xrefs for %s %s\n", te->desc, te->name); + } + + te = te->next; + } + } + + /* + * Clean up & we're done. + */ if (ropt->filename) - ResetOutput(AH, sav); + ResetOutput(AH, sav); + if (ropt->useDB) + { + PQfinish(AH->connection); + AH->connection = NULL; + } } +/* + * Allocate a new RestoreOptions block. + * This is mainly so we can initialize it, but also for future expansion, + */ RestoreOptions* NewRestoreOptions(void) { RestoreOptions* opts; @@ -145,6 +266,11 @@ RestoreOptions* NewRestoreOptions(void) return opts; } +static int _canRestoreBlobs(ArchiveHandle *AH) +{ + return (AH->ropt->useDB && AH->connection); +} + static void _disableTriggers(ArchiveHandle *AH, TocEntry *te, RestoreOptions *ropt) { ahprintf(AH, "-- Disable triggers\n"); @@ -168,7 +294,7 @@ static void _enableTriggers(ArchiveHandle *AH, TocEntry *te, RestoreOptions *rop /* - * This is a routine that is available to pg_dump, hence the 'Archive*' parameter. + * This is a routine that is part of the dumper interface, hence the 'Archive*' parameter. */ /* Public */ @@ -176,6 +302,10 @@ int WriteData(Archive* AHX, const void* data, int dLen) { ArchiveHandle* AH = (ArchiveHandle*)AHX; + if (!AH->currToc) + die_horribly(AH, "%s: WriteData can not be called outside the context of " + "a DataDumper routine\n", progname); + return (*AH->WriteDataPtr)(AH, data, dLen); } @@ -187,7 +317,7 @@ int WriteData(Archive* AHX, const void* data, int dLen) /* Public */ void ArchiveEntry(Archive* AHX, const char* oid, const char* name, const char* desc, const char* (deps[]), const char* defn, - const char* dropStmt, const char* owner, + const char* dropStmt, const char* copyStmt, const char* owner, DataDumperPtr dumpFn, void* dumpArg) { ArchiveHandle* AH = (ArchiveHandle*)AHX; @@ -196,9 +326,9 @@ void ArchiveEntry(Archive* AHX, const char* oid, const char* name, AH->lastID++; AH->tocCount++; - newToc = (TocEntry*)malloc(sizeof(TocEntry)); + newToc = (TocEntry*)calloc(1, sizeof(TocEntry)); if (!newToc) - die_horribly("Archiver: unable to allocate memory for TOC entry\n"); + die_horribly(AH, "Archiver: unable to allocate memory for TOC entry\n"); newToc->prev = AH->toc->prev; newToc->next = AH->toc; @@ -212,6 +342,7 @@ void ArchiveEntry(Archive* AHX, const char* oid, const char* name, newToc->desc = strdup(desc); newToc->defn = strdup(defn); newToc->dropStmt = strdup(dropStmt); + newToc->copyStmt = copyStmt ? strdup(copyStmt) : NULL; newToc->owner = strdup(owner); newToc->printed = 0; newToc->formatData = NULL; @@ -233,10 +364,30 @@ void PrintTOCSummary(Archive* AHX, RestoreOptions *ropt) ArchiveHandle* AH = (ArchiveHandle*) AHX; TocEntry *te = AH->toc->next; OutputContext sav; + char *fmtName; if (ropt->filename) sav = SetOutput(AH, ropt->filename, ropt->compression); + ahprintf(AH, ";\n; Archive created at %s", ctime(&AH->createDate)); + ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", + AH->archdbname, AH->tocCount, AH->compression); + + switch (AH->format) { + case archFiles: + fmtName = "FILES"; + break; + case archCustom: + fmtName = "CUSTOM"; + break; + case archTar: + fmtName = "TAR"; + break; + default: + fmtName = "UNKNOWN"; + } + ahprintf(AH, "; Format: %s\n;\n", fmtName); + ahprintf(AH, ";\n; Selected TOC Entries:\n;\n"); while (te != AH->toc) { @@ -250,12 +401,86 @@ void PrintTOCSummary(Archive* AHX, RestoreOptions *ropt) } /*********** + * BLOB Archival + ***********/ + +/* Called by a dumper to signal start of a BLOB */ +int StartBlob(Archive* AHX, int oid) +{ + ArchiveHandle* AH = (ArchiveHandle*)AHX; + + if (!AH->StartBlobPtr) + die_horribly(AH, "%s: BLOB output not supported in chosen format\n", progname); + + (*AH->StartBlobPtr)(AH, AH->currToc, oid); + + return 1; +} + +/* Called by a dumper to signal end of a BLOB */ +int EndBlob(Archive* AHX, int oid) +{ + ArchiveHandle* AH = (ArchiveHandle*)AHX; + + if (AH->EndBlobPtr) + (*AH->EndBlobPtr)(AH, AH->currToc, oid); + + return 1; +} + +/********** + * BLOB Restoration + **********/ + +/* + * Called by a format handler to initiate restoration of a blob + */ +void StartRestoreBlob(ArchiveHandle* AH, int oid) +{ + int loOid; + + if (!AH->createdBlobXref) + { + if (!AH->connection) + die_horribly(AH, "%s: can not restore BLOBs without a database connection", progname); + + CreateBlobXrefTable(AH); + AH->createdBlobXref = 1; + } + + loOid = lo_creat(AH->connection, INV_READ | INV_WRITE); + if (loOid == 0) + die_horribly(AH, "%s: unable to create BLOB\n", progname); + + ahlog(AH, 1, "Restoring BLOB oid %d as %d\n", oid, loOid); + + StartTransaction(AH); + + InsertBlobXref(AH, oid, loOid); + + AH->loFd = lo_open(AH->connection, loOid, INV_WRITE); + if (AH->loFd == -1) + die_horribly(AH, "%s: unable to open BLOB\n", progname); + + AH->writingBlob = 1; +} + +void EndRestoreBlob(ArchiveHandle* AH, int oid) +{ + lo_close(AH->connection, AH->loFd); + AH->writingBlob = 0; + + CommitTransaction(AH); +} + +/*********** * Sorting and Reordering ***********/ /* * Move TOC entries of the specified type to the START of the TOC. */ + /* Public */ void MoveToStart(Archive* AHX, char *oType) { @@ -356,7 +581,7 @@ void SortTocFromFile(Archive* AHX, RestoreOptions *ropt) /* Setup the file */ fh = fopen(ropt->tocFile, PG_BINARY_R); if (!fh) - die_horribly("%s: could not open TOC file\n", progname); + die_horribly(AH, "%s: could not open TOC file\n", progname); while (fgets(buf, 1024, fh) != NULL) { @@ -377,14 +602,14 @@ void SortTocFromFile(Archive* AHX, RestoreOptions *ropt) id = strtol(buf, &endptr, 10); if (endptr == buf) { - fprintf(stderr, "%s: warning - line ignored: %s\n", progname, buf); + fprintf(stderr, "%s: WARNING - line ignored: %s\n", progname, buf); continue; } /* Find TOC entry */ te = _getTocEntry(AH, id); if (!te) - die_horribly("%s: could not find entry for id %d\n",progname, id); + die_horribly(AH, "%s: could not find entry for id %d\n",progname, id); ropt->idWanted[id-1] = 1; @@ -428,7 +653,7 @@ int archprintf(Archive* AH, const char *fmt, ...) if ((p = malloc(bSize)) == NULL) { va_end(ap); - die_horribly("%s: could not allocate buffer for archprintf\n", progname); + exit_horribly(AH, "%s: could not allocate buffer for archprintf\n", progname); } cnt = vsnprintf(p, bSize, fmt, ap); } @@ -519,15 +744,15 @@ int ahprintf(ArchiveHandle* AH, const char *fmt, ...) /* This is paranoid: deal with the possibility that vsnprintf is willing to ignore trailing null */ /* or returns > 0 even if string does not fit. It may be the case that it returns cnt = bufsize */ while (cnt < 0 || cnt >= (bSize - 1) ) { - if (p != NULL) free(p); - bSize *= 2; - p = (char*)malloc(bSize); - if (p == NULL) - { - va_end(ap); - die_horribly("%s: could not allocate buffer for ahprintf\n", progname); - } - cnt = vsnprintf(p, bSize, fmt, ap); + if (p != NULL) free(p); + bSize *= 2; + p = (char*)malloc(bSize); + if (p == NULL) + { + va_end(ap); + die_horribly(AH, "%s: could not allocate buffer for ahprintf\n", progname); + } + cnt = vsnprintf(p, bSize, fmt, ap); } va_end(ap); ahwrite(p, 1, cnt, AH); @@ -535,28 +760,82 @@ int ahprintf(ArchiveHandle* AH, const char *fmt, ...) return cnt; } +void ahlog(ArchiveHandle* AH, int level, const char *fmt, ...) +{ + va_list ap; + + if (AH->debugLevel < level && (!AH->public.verbose || level > 1)) + return; + + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); +} + /* - * Write buffer to the output file (usually stdout). + * Write buffer to the output file (usually stdout). This is user for + * outputting 'restore' scripts etc. It is even possible for an archive + * format to create a custom output routine to 'fake' a restore if it + * wants to generate a script (see TAR output). */ int ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle* AH) { - if (AH->gzOut) - return GZWRITE((void*)ptr, size, nmemb, AH->OF); - else - return fwrite((void*)ptr, size, nmemb, AH->OF); + int res; + + if (AH->writingBlob) + { + res = lo_write(AH->connection, AH->loFd, (void*)ptr, size * nmemb); + ahlog(AH, 5, "Wrote %d bytes of BLOB data (result = %d)\n", size * nmemb, res); + return res; + } + else if (AH->gzOut) + return GZWRITE((void*)ptr, size, nmemb, AH->OF); + else if (AH->CustomOutPtr) + return AH->CustomOutPtr(AH, ptr, size * nmemb); + else + { + /* + * If we're doing a restore, and it's direct to DB, and we're connected + * then send it to the DB. + */ + if (AH->ropt && AH->ropt->useDB && AH->connection) + return ExecuteSqlCommandBuf(AH, (void*)ptr, size*nmemb); + else + return fwrite((void*)ptr, size, nmemb, AH->OF); + } +} + +/* Common exit code */ +static void _die_horribly(ArchiveHandle *AH, const char *fmt, va_list ap) +{ + vfprintf(stderr, fmt, ap); + + if (AH) + if (AH->connection) + PQfinish(AH->connection); + + exit(1); } +/* External use */ +void exit_horribly(Archive *AH, const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + _die_horribly((ArchiveHandle*)AH, fmt, ap); +} -void die_horribly(const char *fmt, ...) +/* Archiver use (just different arg declaration) */ +void die_horribly(ArchiveHandle *AH, const char *fmt, ...) { va_list ap; va_start(ap, fmt); - vfprintf(stderr, fmt, ap); - va_end(ap); - exit(1); + _die_horribly(AH, fmt, ap); } + static void _moveAfter(ArchiveHandle* AH, TocEntry* pos, TocEntry* te) { te->prev->next = te->next; @@ -622,9 +901,9 @@ int WriteInt(ArchiveHandle* AH, int i) /* SIGN byte */ if (i < 0) { (*AH->WriteBytePtr)(AH, 1); - i = -i; + i = -i; } else { - (*AH->WriteBytePtr)(AH, 0); + (*AH->WriteBytePtr)(AH, 0); } for(b = 0 ; b < AH->intSize ; b++) { @@ -638,30 +917,40 @@ int WriteInt(ArchiveHandle* AH, int i) int ReadInt(ArchiveHandle* AH) { int res = 0; - int shft = 1; int bv, b; int sign = 0; /* Default positive */ + int bitShift = 0; if (AH->version > K_VERS_1_0) - /* Read a sign byte */ - sign = (*AH->ReadBytePtr)(AH); + /* Read a sign byte */ + sign = (*AH->ReadBytePtr)(AH); for( b = 0 ; b < AH->intSize ; b++) { - bv = (*AH->ReadBytePtr)(AH); - res = res + shft * bv; - shft *= 256; + bv = (*AH->ReadBytePtr)(AH) & 0xFF; + if (bv != 0) + res = res + (bv << bitShift); + bitShift += 8; } if (sign) - res = - res; + res = - res; return res; } int WriteStr(ArchiveHandle* AH, char* c) { - int l = WriteInt(AH, strlen(c)); - return (*AH->WriteBufPtr)(AH, c, strlen(c)) + l; + int res; + + if (c) + { + res = WriteInt(AH, strlen(c)); + res += (*AH->WriteBufPtr)(AH, c, strlen(c)); + } + else + res = WriteInt(AH, -1); + + return res; } char* ReadStr(ArchiveHandle* AH) @@ -670,12 +959,18 @@ char* ReadStr(ArchiveHandle* AH) int l; l = ReadInt(AH); - buf = (char*)malloc(l+1); - if (!buf) - die_horribly("Archiver: Unable to allocate sufficient memory in ReadStr\n"); + if (l == -1) + buf = NULL; + else + { + buf = (char*)malloc(l+1); + if (!buf) + die_horribly(AH, "%s: Unable to allocate sufficient memory in ReadStr - " "requested %d (0x%x) bytes\n", progname, l, l); + + (*AH->ReadBufPtr)(AH, (void*)buf, l); + buf[l] = '\0'; + } - (*AH->ReadBufPtr)(AH, (void*)buf, l); - buf[l] = '\0'; return buf; } @@ -686,138 +981,188 @@ int _discoverArchiveFormat(ArchiveHandle* AH) int cnt; int wantClose = 0; + /* + * fprintf(stderr, "%s: Attempting to ascertain archive format\n", progname); + */ + + if (AH->lookahead) + free(AH->lookahead); + + AH->lookaheadSize = 512; + AH->lookahead = calloc(1, 512); + AH->lookaheadLen = 0; + AH->lookaheadPos = 0; if (AH->fSpec) { - wantClose = 1; - fh = fopen(AH->fSpec, PG_BINARY_R); + wantClose = 1; + fh = fopen(AH->fSpec, PG_BINARY_R); } else { - fh = stdin; + fh = stdin; } if (!fh) - die_horribly("Archiver: could not open input file\n"); + die_horribly(AH, "Archiver: could not open input file\n"); cnt = fread(sig, 1, 5, fh); if (cnt != 5) - die_horribly("%s: input file is too short, or is unreadable\n", progname); + die_horribly(AH, "%s: input file is too short, or is unreadable\n", progname); - if (strncmp(sig, "PGDMP", 5) != 0) - die_horribly("%s: input file does not appear to be a valid archive\n", progname); + /* Save it, just in case we need it later*/ + strncpy(&AH->lookahead[0], sig, 5); + AH->lookaheadLen = 5; - AH->vmaj = fgetc(fh); - AH->vmin = fgetc(fh); + if (strncmp(sig, "PGDMP", 5) == 0) + { + AH->vmaj = fgetc(fh); + AH->vmin = fgetc(fh); + + /* Save these too... */ + AH->lookahead[AH->lookaheadLen++] = AH->vmaj; + AH->lookahead[AH->lookaheadLen++] = AH->vmin; + + /* Check header version; varies from V1.0 */ + if (AH->vmaj > 1 || ( (AH->vmaj == 1) && (AH->vmin > 0) ) ) /* Version > 1.0 */ + { + AH->vrev = fgetc(fh); + AH->lookahead[AH->lookaheadLen++] = AH->vrev; + } + else + AH->vrev = 0; + + AH->intSize = fgetc(fh); + AH->lookahead[AH->lookaheadLen++] = AH->intSize; + + AH->format = fgetc(fh); + AH->lookahead[AH->lookaheadLen++] = AH->format; + + /* Make a convenient integer 00 */ + AH->version = ( (AH->vmaj * 256 + AH->vmin) * 256 + AH->vrev ) * 256 + 0; + } else { + /* + * *Maybe* we have a tar archive format file... + * So, read first 512 byte header... + */ + cnt = fread(&AH->lookahead[AH->lookaheadLen], 1, 512 - AH->lookaheadLen, fh); + AH->lookaheadLen += cnt; - /* Check header version; varies from V1.0 */ - if (AH->vmaj > 1 || ( (AH->vmaj == 1) && (AH->vmin > 0) ) ) /* Version > 1.0 */ - AH->vrev = fgetc(fh); - else - AH->vrev = 0; + if (AH->lookaheadLen != 512) + die_horribly(AH, "%s: input file does not appear to be a valid archive (too short?)\n", + progname); - AH->intSize = fgetc(fh); - AH->format = fgetc(fh); + if (!isValidTarHeader(AH->lookahead)) + die_horribly(AH, "%s: input file does not appear to be a valid archive\n", progname); - /* Make a convenient integer 00 */ - AH->version = ( (AH->vmaj * 256 + AH->vmin) * 256 + AH->vrev ) * 256 + 0; + AH->format = archTar; + } /* If we can't seek, then mark the header as read */ if (fseek(fh, 0, SEEK_SET) != 0) - AH->readHeader = 1; + { + /* + * NOTE: Formats that use the looahead buffer can unset this in their Init routine. + */ + AH->readHeader = 1; + } + else + AH->lookaheadLen = 0; /* Don't bother since we've reset the file */ + + /* + *fprintf(stderr, "%s: read %d bytes into lookahead buffer\n", progname, AH->lookaheadLen); + */ /* Close the file */ if (wantClose) - fclose(fh); + fclose(fh); return AH->format; - } /* * Allocate an archive handle */ -static ArchiveHandle* _allocAH(const char* FileSpec, ArchiveFormat fmt, - int compression, ArchiveMode mode) { +static ArchiveHandle* _allocAH(const char* FileSpec, const ArchiveFormat fmt, + const int compression, ArchiveMode mode) +{ ArchiveHandle* AH; + /* + *fprintf(stderr, "%s: allocating AH for %s, format %d\n", progname, FileSpec, fmt); + */ + AH = (ArchiveHandle*)calloc(1, sizeof(ArchiveHandle)); if (!AH) - die_horribly("Archiver: Could not allocate archive handle\n"); + die_horribly(AH, "Archiver: Could not allocate archive handle\n"); AH->vmaj = K_VERS_MAJOR; AH->vmin = K_VERS_MINOR; + AH->createDate = time(NULL); + AH->intSize = sizeof(int); AH->lastID = 0; if (FileSpec) { - AH->fSpec = strdup(FileSpec); + AH->fSpec = strdup(FileSpec); + /* + * Not used; maybe later.... + * + * AH->workDir = strdup(FileSpec); + * for(i=strlen(FileSpec) ; i > 0 ; i--) + * if (AH->workDir[i-1] == '/') + */ } else { - AH->fSpec = NULL; - } - AH->FH = NULL; - AH->formatData = NULL; + AH->fSpec = NULL; + } - AH->currToc = NULL; AH->currUser = ""; AH->toc = (TocEntry*)calloc(1, sizeof(TocEntry)); if (!AH->toc) - die_horribly("Archiver: Could not allocate TOC header\n"); + die_horribly(AH, "Archiver: Could not allocate TOC header\n"); - AH->tocCount = 0; AH->toc->next = AH->toc; AH->toc->prev = AH->toc; - AH->toc->id = 0; - AH->toc->oid = NULL; - AH->toc->name = NULL; /* eg. MY_SPECIAL_FUNCTION */ - AH->toc->desc = NULL; /* eg. FUNCTION */ - AH->toc->defn = NULL; /* ie. sql to define it */ - AH->toc->depOid = NULL; AH->mode = mode; - AH->format = fmt; AH->compression = compression; - AH->ArchiveEntryPtr = NULL; - - AH->StartDataPtr = NULL; - AH->WriteDataPtr = NULL; - AH->EndDataPtr = NULL; - - AH->WriteBytePtr = NULL; - AH->ReadBytePtr = NULL; - AH->WriteBufPtr = NULL; - AH->ReadBufPtr = NULL; - AH->ClosePtr = NULL; - AH->WriteExtraTocPtr = NULL; - AH->ReadExtraTocPtr = NULL; - AH->PrintExtraTocPtr = NULL; - - AH->readHeader = 0; + AH->pgCopyBuf = createPQExpBuffer(); + AH->sqlBuf = createPQExpBuffer(); /* Open stdout with no compression for AH output handle */ AH->gzOut = 0; AH->OF = stdout; + /* + *fprintf(stderr, "%s: archive format is %d\n", progname, fmt); + */ + if (fmt == archUnknown) - fmt = _discoverArchiveFormat(AH); + AH->format = _discoverArchiveFormat(AH); + else + AH->format = fmt; - switch (fmt) { + switch (AH->format) { - case archCustom: - InitArchiveFmt_Custom(AH); - break; + case archCustom: + InitArchiveFmt_Custom(AH); + break; - case archFiles: - InitArchiveFmt_Files(AH); - break; + case archFiles: + InitArchiveFmt_Files(AH); + break; - case archPlainText: - InitArchiveFmt_PlainText(AH); - break; + case archNull: + InitArchiveFmt_Null(AH); + break; - default: - die_horribly("Archiver: Unrecognized file format '%d'\n", fmt); + case archTar: + InitArchiveFmt_Tar(AH); + break; + + default: + die_horribly(AH, "Archiver: Unrecognized file format '%d'\n", fmt); } return AH; @@ -827,13 +1172,25 @@ static ArchiveHandle* _allocAH(const char* FileSpec, ArchiveFormat fmt, void WriteDataChunks(ArchiveHandle* AH) { TocEntry *te = AH->toc->next; + StartDataPtr startPtr; + EndDataPtr endPtr; while (te != AH->toc) { if (te->dataDumper != NULL) { AH->currToc = te; /* printf("Writing data for %d (%x)\n", te->id, te); */ - if (AH->StartDataPtr != NULL) { - (*AH->StartDataPtr)(AH, te); + + if (strcmp(te->desc, "BLOBS") == 0) + { + startPtr = AH->StartBlobsPtr; + endPtr = AH->EndBlobsPtr; + } else { + startPtr = AH->StartDataPtr; + endPtr = AH->EndDataPtr; + } + + if (startPtr != NULL) { + (*startPtr)(AH, te); } /* printf("Dumper arg for %d is %x\n", te->id, te->dataDumperArg); */ @@ -842,12 +1199,12 @@ void WriteDataChunks(ArchiveHandle* AH) */ (*te->dataDumper)((Archive*)AH, te->oid, te->dataDumperArg); - if (AH->EndDataPtr != NULL) { - (*AH->EndDataPtr)(AH, te); + if (endPtr != NULL) { + (*endPtr)(AH, te); } AH->currToc = NULL; } - te = te->next; + te = te->next; } } @@ -866,6 +1223,7 @@ void WriteToc(ArchiveHandle* AH) WriteStr(AH, te->desc); WriteStr(AH, te->defn); WriteStr(AH, te->dropStmt); + WriteStr(AH, te->copyStmt); WriteStr(AH, te->owner); if (AH->WriteExtraTocPtr) { (*AH->WriteExtraTocPtr)(AH, te); @@ -884,71 +1242,79 @@ void ReadToc(ArchiveHandle* AH) for( i = 0 ; i < AH->tocCount ; i++) { - te = (TocEntry*)malloc(sizeof(TocEntry)); - te->id = ReadInt(AH); - - /* Sanity check */ - if (te->id <= 0 || te->id > AH->tocCount) - die_horribly("Archiver: failed sanity check (bad entry id) - perhaps a corrupt TOC\n"); - - te->hadDumper = ReadInt(AH); - te->oid = ReadStr(AH); - te->oidVal = atoi(te->oid); - te->name = ReadStr(AH); - te->desc = ReadStr(AH); - te->defn = ReadStr(AH); - te->dropStmt = ReadStr(AH); - te->owner = ReadStr(AH); - if (AH->ReadExtraTocPtr) { - (*AH->ReadExtraTocPtr)(AH, te); - } - te->prev = AH->toc->prev; - AH->toc->prev->next = te; - AH->toc->prev = te; - te->next = AH->toc; + te = (TocEntry*)calloc(1, sizeof(TocEntry)); + te->id = ReadInt(AH); + + /* Sanity check */ + if (te->id <= 0 || te->id > AH->tocCount) + die_horribly(AH, "Archiver: failed sanity check (bad entry id) - perhaps a corrupt TOC\n"); + + te->hadDumper = ReadInt(AH); + te->oid = ReadStr(AH); + te->oidVal = atoi(te->oid); + te->name = ReadStr(AH); + te->desc = ReadStr(AH); + te->defn = ReadStr(AH); + te->dropStmt = ReadStr(AH); + + if (AH->version >= K_VERS_1_3) + te->copyStmt = ReadStr(AH); + + te->owner = ReadStr(AH); + + if (AH->ReadExtraTocPtr) { + (*AH->ReadExtraTocPtr)(AH, te); + } + + ahlog(AH, 3, "Read TOC entry %d (id %d) for %s %s\n", i, te->id, te->desc, te->name); + + te->prev = AH->toc->prev; + AH->toc->prev->next = te; + AH->toc->prev = te; + te->next = AH->toc; } } static int _tocEntryRequired(TocEntry* te, RestoreOptions *ropt) { - int res = 3; /* Data and Schema */ - + int res = 3; /* Schema = 1, Data = 2, Both = 3 */ + /* If it's an ACL, maybe ignore it */ if (ropt->aclsSkip && strcmp(te->desc,"ACL") == 0) - return 0; + return 0; /* Check if tablename only is wanted */ if (ropt->selTypes) { - if ( (strcmp(te->desc, "TABLE") == 0) || (strcmp(te->desc, "TABLE DATA") == 0) ) - { - if (!ropt->selTable) - return 0; - if (ropt->tableNames && strcmp(ropt->tableNames, te->name) != 0) - return 0; - } else if (strcmp(te->desc, "INDEX") == 0) { - if (!ropt->selIndex) - return 0; - if (ropt->indexNames && strcmp(ropt->indexNames, te->name) != 0) - return 0; - } else if (strcmp(te->desc, "FUNCTION") == 0) { - if (!ropt->selFunction) - return 0; - if (ropt->functionNames && strcmp(ropt->functionNames, te->name) != 0) - return 0; - } else if (strcmp(te->desc, "TRIGGER") == 0) { - if (!ropt->selTrigger) - return 0; - if (ropt->triggerNames && strcmp(ropt->triggerNames, te->name) != 0) - return 0; - } else { - return 0; + if ( (strcmp(te->desc, "TABLE") == 0) || (strcmp(te->desc, "TABLE DATA") == 0) ) + { + if (!ropt->selTable) + return 0; + if (ropt->tableNames && strcmp(ropt->tableNames, te->name) != 0) + return 0; + } else if (strcmp(te->desc, "INDEX") == 0) { + if (!ropt->selIndex) + return 0; + if (ropt->indexNames && strcmp(ropt->indexNames, te->name) != 0) + return 0; + } else if (strcmp(te->desc, "FUNCTION") == 0) { + if (!ropt->selFunction) + return 0; + if (ropt->functionNames && strcmp(ropt->functionNames, te->name) != 0) + return 0; + } else if (strcmp(te->desc, "TRIGGER") == 0) { + if (!ropt->selTrigger) + return 0; + if (ropt->triggerNames && strcmp(ropt->triggerNames, te->name) != 0) + return 0; + } else { + return 0; + } } - } /* Mask it if we only want schema */ if (ropt->schemaOnly) - res = res & 1; + res = res & 1; /* Mask it we only want data */ if (ropt->dataOnly) @@ -956,15 +1322,15 @@ static int _tocEntryRequired(TocEntry* te, RestoreOptions *ropt) /* Mask it if we don't have a schema contribition */ if (!te->defn || strlen(te->defn) == 0) - res = res & 2; + res = res & 2; /* Mask it if we don't have a possible data contribition */ if (!te->hadDumper) - res = res & 1; + res = res & 1; /* Finally, if we used a list, limit based on that as well */ if (ropt->limitToList && !ropt->idWanted[te->id - 1]) - return 0; + return 0; return res; } @@ -979,8 +1345,9 @@ static int _printTocEntry(ArchiveHandle* AH, TocEntry* te, RestoreOptions *ropt) ahprintf(AH, "--\n\n"); if (te->owner && strlen(te->owner) != 0 && strcmp(AH->currUser, te->owner) != 0) { - ahprintf(AH, "\\connect - %s\n", te->owner); - AH->currUser = te->owner; + //todo pjw - fix for db connection... + //ahprintf(AH, "\\connect - %s\n", te->owner); + AH->currUser = te->owner; } ahprintf(AH, "%s\n", te->defn); @@ -990,6 +1357,8 @@ static int _printTocEntry(ArchiveHandle* AH, TocEntry* te, RestoreOptions *ropt) void WriteHead(ArchiveHandle* AH) { + struct tm crtm; + (*AH->WriteBufPtr)(AH, "PGDMP", 5); /* Magic code */ (*AH->WriteBytePtr)(AH, AH->vmaj); (*AH->WriteBytePtr)(AH, AH->vmin); @@ -1003,70 +1372,102 @@ void WriteHead(ArchiveHandle* AH) "archive will be uncompressed \n", progname); AH->compression = 0; - (*AH->WriteBytePtr)(AH, 0); -#else - - (*AH->WriteBytePtr)(AH, AH->compression); +#endif -#endif + WriteInt(AH, AH->compression); + + crtm = *localtime(&AH->createDate); + WriteInt(AH, crtm.tm_sec); + WriteInt(AH, crtm.tm_min); + WriteInt(AH, crtm.tm_hour); + WriteInt(AH, crtm.tm_mday); + WriteInt(AH, crtm.tm_mon); + WriteInt(AH, crtm.tm_year); + WriteInt(AH, crtm.tm_isdst); + WriteStr(AH, AH->dbname); } void ReadHead(ArchiveHandle* AH) { - char tmpMag[7]; - int fmt; + char tmpMag[7]; + int fmt; + struct tm crtm; + /* If we haven't already read the header... */ if (!AH->readHeader) { - (*AH->ReadBufPtr)(AH, tmpMag, 5); + (*AH->ReadBufPtr)(AH, tmpMag, 5); - if (strncmp(tmpMag,"PGDMP", 5) != 0) - die_horribly("Archiver: Did not fing magic PGDMP in file header\n"); + if (strncmp(tmpMag,"PGDMP", 5) != 0) + die_horribly(AH, "Archiver: Did not fing magic PGDMP in file header\n"); - AH->vmaj = (*AH->ReadBytePtr)(AH); - AH->vmin = (*AH->ReadBytePtr)(AH); + AH->vmaj = (*AH->ReadBytePtr)(AH); + AH->vmin = (*AH->ReadBytePtr)(AH); - if (AH->vmaj > 1 || ( (AH->vmaj == 1) && (AH->vmin > 0) ) ) /* Version > 1.0 */ - { - AH->vrev = (*AH->ReadBytePtr)(AH); - } else { - AH->vrev = 0; - } + if (AH->vmaj > 1 || ( (AH->vmaj == 1) && (AH->vmin > 0) ) ) /* Version > 1.0 */ + { + AH->vrev = (*AH->ReadBytePtr)(AH); + } else { + AH->vrev = 0; + } - AH->version = ( (AH->vmaj * 256 + AH->vmin) * 256 + AH->vrev ) * 256 + 0; + AH->version = ( (AH->vmaj * 256 + AH->vmin) * 256 + AH->vrev ) * 256 + 0; - if (AH->version < K_VERS_1_0 || AH->version > K_VERS_MAX) - die_horribly("Archiver: unsupported version (%d.%d) in file header\n", AH->vmaj, AH->vmin); + if (AH->version < K_VERS_1_0 || AH->version > K_VERS_MAX) + die_horribly(AH, "%s: unsupported version (%d.%d) in file header\n", + progname, AH->vmaj, AH->vmin); - AH->intSize = (*AH->ReadBytePtr)(AH); - if (AH->intSize > 32) - die_horribly("Archiver: sanity check on integer size (%d) failes\n", AH->intSize); + AH->intSize = (*AH->ReadBytePtr)(AH); + if (AH->intSize > 32) + die_horribly(AH, "Archiver: sanity check on integer size (%d) failes\n", AH->intSize); - if (AH->intSize > sizeof(int)) - fprintf(stderr, "\nWARNING: Backup file was made on a machine with larger integers, " - "some operations may fail\n"); + if (AH->intSize > sizeof(int)) + fprintf(stderr, "\n%s: WARNING - archive was made on a machine with larger integers, " + "some operations may fail\n", progname); - fmt = (*AH->ReadBytePtr)(AH); + fmt = (*AH->ReadBytePtr)(AH); - if (AH->format != fmt) - die_horribly("Archiver: expected format (%d) differs from format found in file (%d)\n", - AH->format, fmt); + if (AH->format != fmt) + die_horribly(AH, "%s: expected format (%d) differs from format found in file (%d)\n", + progname, AH->format, fmt); } if (AH->version >= K_VERS_1_2) { - AH->compression = (*AH->ReadBytePtr)(AH); + if (AH->version < K_VERS_1_4) + AH->compression = (*AH->ReadBytePtr)(AH); + else + AH->compression = ReadInt(AH); } else { - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compression = Z_DEFAULT_COMPRESSION; } #ifndef HAVE_LIBZ if (AH->compression != 0) - fprintf(stderr, "%s: WARNING - archive is compressed - any data will not be available\n", progname); + fprintf(stderr, "%s: WARNING - archive is compressed - any data will not be available\n", + progname); #endif + if (AH->version >= K_VERS_1_4) + { + crtm.tm_sec = ReadInt(AH); + crtm.tm_min = ReadInt(AH); + crtm.tm_hour = ReadInt(AH); + crtm.tm_mday = ReadInt(AH); + crtm.tm_mon = ReadInt(AH); + crtm.tm_year = ReadInt(AH); + crtm.tm_isdst = ReadInt(AH); + + AH->archdbname = ReadStr(AH); + + AH->createDate = mktime(&crtm); + + if (AH->createDate == (time_t)-1) + fprintf(stderr, "%s: WARNING - bad creation date in header\n", progname); + } + } @@ -1144,5 +1545,12 @@ static int _tocSortCompareByIDNum(const void* p1, const void* p2) } } - +/* + * Maybe I can use this somewhere... + * + *create table pgdump_blob_path(p text); + *insert into pgdump_blob_path values('/home/pjw/work/postgresql-cvs/pgsql/src/bin/pg_dump_140'); + * + *insert into dump_blob_xref select 12345,lo_import(p || '/q.q') from pgdump_blob_path; + */ diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index f34e158409..f3b0ba5385 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -29,6 +29,10 @@ #define __PG_BACKUP_ARCHIVE__ #include +#include + +#include "postgres.h" +#include "pqexpbuffer.h" #ifdef HAVE_LIBZ #include @@ -51,15 +55,23 @@ typedef z_stream *z_streamp; #endif #include "pg_backup.h" +#include "libpq-fe.h" #define K_VERS_MAJOR 1 -#define K_VERS_MINOR 2 -#define K_VERS_REV 2 +#define K_VERS_MINOR 4 +#define K_VERS_REV 3 + +/* Data block types */ +#define BLK_DATA 1 +#define BLK_BLOB 2 +#define BLK_BLOBS 3 /* Some important version numbers (checked in code) */ #define K_VERS_1_0 (( (1 * 256 + 0) * 256 + 0) * 256 + 0) -#define K_VERS_1_2 (( (1 * 256 + 2) * 256 + 0) * 256 + 0) -#define K_VERS_MAX (( (1 * 256 + 2) * 256 + 255) * 256 + 0) +#define K_VERS_1_2 (( (1 * 256 + 2) * 256 + 0) * 256 + 0) /* Allow No ZLIB */ +#define K_VERS_1_3 (( (1 * 256 + 3) * 256 + 0) * 256 + 0) /* BLOBs */ +#define K_VERS_1_4 (( (1 * 256 + 4) * 256 + 0) * 256 + 0) /* Date & name in header */ +#define K_VERS_MAX (( (1 * 256 + 4) * 256 + 255) * 256 + 0) struct _archiveHandle; struct _tocEntry; @@ -72,10 +84,15 @@ typedef void (*StartDataPtr) (struct _archiveHandle* AH, struct _tocEntry* te); typedef int (*WriteDataPtr) (struct _archiveHandle* AH, const void* data, int dLen); typedef void (*EndDataPtr) (struct _archiveHandle* AH, struct _tocEntry* te); -typedef int (*WriteBytePtr) (struct _archiveHandle* AH, const int i); +typedef void (*StartBlobsPtr) (struct _archiveHandle* AH, struct _tocEntry* te); +typedef void (*StartBlobPtr) (struct _archiveHandle* AH, struct _tocEntry* te, int oid); +typedef void (*EndBlobPtr) (struct _archiveHandle* AH, struct _tocEntry* te, int oid); +typedef void (*EndBlobsPtr) (struct _archiveHandle* AH, struct _tocEntry* te); + +typedef int (*WriteBytePtr) (struct _archiveHandle* AH, const int i); typedef int (*ReadBytePtr) (struct _archiveHandle* AH); -typedef int (*WriteBufPtr) (struct _archiveHandle* AH, const void* c, int len); -typedef int (*ReadBufPtr) (struct _archiveHandle* AH, void* buf, int len); +typedef int (*WriteBufPtr) (struct _archiveHandle* AH, const void* c, int len); +typedef int (*ReadBufPtr) (struct _archiveHandle* AH, void* buf, int len); typedef void (*SaveArchivePtr) (struct _archiveHandle* AH); typedef void (*WriteExtraTocPtr) (struct _archiveHandle* AH, struct _tocEntry* te); typedef void (*ReadExtraTocPtr) (struct _archiveHandle* AH, struct _tocEntry* te); @@ -83,6 +100,8 @@ typedef void (*PrintExtraTocPtr) (struct _archiveHandle* AH, struct _tocEntry* t typedef void (*PrintTocDataPtr) (struct _archiveHandle* AH, struct _tocEntry* te, RestoreOptions *ropt); +typedef int (*CustomOutPtr) (struct _archiveHandle* AH, const void* buf, int len); + typedef int (*TocSortCompareFn) (const void* te1, const void *te2); typedef enum _archiveMode { @@ -95,16 +114,44 @@ typedef struct _outputContext { int gzOut; } OutputContext; +typedef enum { + SQL_SCAN = 0, + SQL_IN_SQL_COMMENT, + SQL_IN_EXT_COMMENT, + SQL_IN_QUOTE} sqlparseState; + +typedef struct { + int backSlash; + sqlparseState state; + char lastChar; + char quoteChar; +} sqlparseInfo; + typedef struct _archiveHandle { + Archive public; /* Public part of archive */ char vmaj; /* Version of file */ char vmin; char vrev; int version; /* Conveniently formatted version */ + int debugLevel; /* Not used. Intended for logging */ int intSize; /* Size of an integer in the archive */ ArchiveFormat format; /* Archive format */ + sqlparseInfo sqlparse; + PQExpBuffer sqlBuf; + + time_t createDate; /* Date archive created */ + + /* + * Fields used when discovering header. + * A format can always get the previous read bytes from here... + */ int readHeader; /* Used if file header has been read already */ + char *lookahead; /* Buffer used when reading header to discover format */ + int lookaheadSize; /* Size of allocated buffer */ + int lookaheadLen; /* Length of data in lookahead */ + int lookaheadPos; /* Current read position in lookahead buffer */ ArchiveEntryPtr ArchiveEntryPtr; /* Called for each metadata object */ StartDataPtr StartDataPtr; /* Called when table data is about to be dumped */ @@ -121,11 +168,33 @@ typedef struct _archiveHandle { PrintExtraTocPtr PrintExtraTocPtr; /* Extra TOC info for format */ PrintTocDataPtr PrintTocDataPtr; - int lastID; /* Last internal ID for a TOC entry */ - char* fSpec; /* Archive File Spec */ - FILE *FH; /* General purpose file handle */ - void *OF; - int gzOut; /* Output file */ + StartBlobsPtr StartBlobsPtr; + EndBlobsPtr EndBlobsPtr; + StartBlobPtr StartBlobPtr; + EndBlobPtr EndBlobPtr; + + CustomOutPtr CustomOutPtr; /* Alternate script output routine */ + + /* Stuff for direct DB connection */ + char username[100]; + char *dbname; /* Name of db for connection */ + char *archdbname; /* DB name *read* from archive */ + char *pghost; + char *pgport; + PGconn *connection; + int connectToDB; /* Flag to indicate if direct DB connection is required */ + int pgCopyIn; /* Currently in libpq 'COPY IN' mode. */ + PQExpBuffer pgCopyBuf; /* Left-over data from incomplete lines in COPY IN */ + + int loFd; /* BLOB fd */ + int writingBlob; /* Flag */ + int createdBlobXref; /* Flag */ + + int lastID; /* Last internal ID for a TOC entry */ + char* fSpec; /* Archive File Spec */ + FILE *FH; /* General purpose file handle */ + void *OF; + int gzOut; /* Output file */ struct _tocEntry* toc; /* List of TOC entries */ int tocCount; /* Number of TOC entries */ @@ -135,6 +204,7 @@ typedef struct _archiveHandle { ArchiveMode mode; /* File mode - r or w */ void* formatData; /* Header data specific to file format */ + RestoreOptions *ropt; /* Used to check restore options in ahwrite etc */ } ArchiveHandle; typedef struct _tocEntry { @@ -148,6 +218,7 @@ typedef struct _tocEntry { char* desc; char* defn; char* dropStmt; + char* copyStmt; char* owner; char** depOid; int printed; /* Indicates if entry defn has been dumped */ @@ -159,7 +230,8 @@ typedef struct _tocEntry { } TocEntry; -extern void die_horribly(const char *fmt, ...); +/* Used everywhere */ +extern void die_horribly(ArchiveHandle *AH, const char *fmt, ...); extern void WriteTOC(ArchiveHandle* AH); extern void ReadTOC(ArchiveHandle* AH); @@ -175,19 +247,27 @@ extern int TocIDRequired(ArchiveHandle* AH, int id, RestoreOptions *ropt); * Mandatory routines for each supported format */ -extern int WriteInt(ArchiveHandle* AH, int i); -extern int ReadInt(ArchiveHandle* AH); -extern char* ReadStr(ArchiveHandle* AH); -extern int WriteStr(ArchiveHandle* AH, char* s); +extern int WriteInt(ArchiveHandle* AH, int i); +extern int ReadInt(ArchiveHandle* AH); +extern char* ReadStr(ArchiveHandle* AH); +extern int WriteStr(ArchiveHandle* AH, char* s); + +extern void StartRestoreBlob(ArchiveHandle* AH, int oid); +extern void EndRestoreBlob(ArchiveHandle* AH, int oid); -extern void InitArchiveFmt_Custom(ArchiveHandle* AH); -extern void InitArchiveFmt_Files(ArchiveHandle* AH); -extern void InitArchiveFmt_PlainText(ArchiveHandle* AH); +extern void InitArchiveFmt_Custom(ArchiveHandle* AH); +extern void InitArchiveFmt_Files(ArchiveHandle* AH); +extern void InitArchiveFmt_Null(ArchiveHandle* AH); +extern void InitArchiveFmt_Tar(ArchiveHandle* AH); + +extern int isValidTarHeader(char *header); extern OutputContext SetOutput(ArchiveHandle* AH, char *filename, int compression); -extern void ResetOutput(ArchiveHandle* AH, OutputContext savedContext); +extern void ResetOutput(ArchiveHandle* AH, OutputContext savedContext); int ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle* AH); int ahprintf(ArchiveHandle* AH, const char *fmt, ...); +void ahlog(ArchiveHandle* AH, int level, const char *fmt, ...); + #endif diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 3edbb751f9..f5b208e233 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -1,584 +1,935 @@ -/*------------------------------------------------------------------------- - * - * pg_backup_custom.c - * - * Implements the custom output format. - * - * See the headers to pg_restore for more details. - * - * Copyright (c) 2000, Philip Warner - * Rights are granted to use this software in any way so long - * as this notice is not removed. - * - * The author is not responsible for loss or damages that may - * result from it's use. - * - * - * IDENTIFICATION - * - * Modifications - 28-Jun-2000 - pjw@rhyme.com.au - * - * Initial version. - * - *------------------------------------------------------------------------- - */ - -#include -#include "pg_backup.h" -#include "pg_backup_archiver.h" - -extern int errno; - -static void _ArchiveEntry(ArchiveHandle* AH, TocEntry* te); -static void _StartData(ArchiveHandle* AH, TocEntry* te); -static int _WriteData(ArchiveHandle* AH, const void* data, int dLen); -static void _EndData(ArchiveHandle* AH, TocEntry* te); -static int _WriteByte(ArchiveHandle* AH, const int i); -static int _ReadByte(ArchiveHandle* ); -static int _WriteBuf(ArchiveHandle* AH, const void* buf, int len); -static int _ReadBuf(ArchiveHandle* AH, void* buf, int len); -static void _CloseArchive(ArchiveHandle* AH); -static void _PrintTocData(ArchiveHandle* AH, TocEntry* te, RestoreOptions *ropt); -static void _WriteExtraToc(ArchiveHandle* AH, TocEntry* te); -static void _ReadExtraToc(ArchiveHandle* AH, TocEntry* te); -static void _PrintExtraToc(ArchiveHandle* AH, TocEntry* te); - -static void _PrintData(ArchiveHandle* AH); -static void _skipData(ArchiveHandle* AH); - -#define zlibOutSize 4096 -#define zlibInSize 4096 - -typedef struct { - z_streamp zp; - char* zlibOut; - char* zlibIn; - int inSize; - int hasSeek; - int filePos; - int dataStart; -} lclContext; - -typedef struct { - int dataPos; - int dataLen; -} lclTocEntry; - -static int _getFilePos(ArchiveHandle* AH, lclContext* ctx); - -static char* progname = "Archiver(custom)"; - -/* - * Handler functions. - */ -void InitArchiveFmt_Custom(ArchiveHandle* AH) -{ - lclContext* ctx; - - /* Assuming static functions, this can be copied for each format. */ - AH->ArchiveEntryPtr = _ArchiveEntry; - AH->StartDataPtr = _StartData; - AH->WriteDataPtr = _WriteData; - AH->EndDataPtr = _EndData; - AH->WriteBytePtr = _WriteByte; - AH->ReadBytePtr = _ReadByte; - AH->WriteBufPtr = _WriteBuf; - AH->ReadBufPtr = _ReadBuf; - AH->ClosePtr = _CloseArchive; - AH->PrintTocDataPtr = _PrintTocData; - AH->ReadExtraTocPtr = _ReadExtraToc; - AH->WriteExtraTocPtr = _WriteExtraToc; - AH->PrintExtraTocPtr = _PrintExtraToc; - - /* - * Set up some special context used in compressing data. - */ - ctx = (lclContext*)malloc(sizeof(lclContext)); - if (ctx == NULL) - die_horribly("%s: Unable to allocate archive context",progname); - AH->formatData = (void*)ctx; - - ctx->zp = (z_streamp)malloc(sizeof(z_stream)); - if (ctx->zp == NULL) - die_horribly("%s: unable to allocate zlib stream archive context",progname); - - ctx->zlibOut = (char*)malloc(zlibOutSize); - ctx->zlibIn = (char*)malloc(zlibInSize); - ctx->inSize = zlibInSize; - ctx->filePos = 0; - - if (ctx->zlibOut == NULL || ctx->zlibIn == NULL) - die_horribly("%s: unable to allocate buffers in archive context",progname); - - /* - * Now open the file - */ - if (AH->mode == archModeWrite) { - if (AH->fSpec && strcmp(AH->fSpec,"") != 0) { - AH->FH = fopen(AH->fSpec, PG_BINARY_W); - } else { - AH->FH = stdout; - } - - if (!AH) - die_horribly("%s: unable to open archive file %s",progname, AH->fSpec); - - ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0); - - } else { - if (AH->fSpec && strcmp(AH->fSpec,"") != 0) { - AH->FH = fopen(AH->fSpec, PG_BINARY_R); - } else { - AH->FH = stdin; - } - if (!AH) - die_horribly("%s: unable to open archive file %s",progname, AH->fSpec); - - ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0); - - ReadHead(AH); - ReadToc(AH); - ctx->dataStart = _getFilePos(AH, ctx); - } - -} - -/* - * - Start a new TOC entry -*/ -static void _ArchiveEntry(ArchiveHandle* AH, TocEntry* te) -{ - lclTocEntry* ctx; - - ctx = (lclTocEntry*)malloc(sizeof(lclTocEntry)); - if (te->dataDumper) { - ctx->dataPos = -1; - } else { - ctx->dataPos = 0; - } - ctx->dataLen = 0; - te->formatData = (void*)ctx; - -} - -static void _WriteExtraToc(ArchiveHandle* AH, TocEntry* te) -{ - lclTocEntry* ctx = (lclTocEntry*)te->formatData; - - WriteInt(AH, ctx->dataPos); - WriteInt(AH, ctx->dataLen); -} - -static void _ReadExtraToc(ArchiveHandle* AH, TocEntry* te) -{ - lclTocEntry* ctx = (lclTocEntry*)te->formatData; - - if (ctx == NULL) { - ctx = (lclTocEntry*)malloc(sizeof(lclTocEntry)); - te->formatData = (void*)ctx; - } - - ctx->dataPos = ReadInt( AH ); - ctx->dataLen = ReadInt( AH ); - -} - -static void _PrintExtraToc(ArchiveHandle* AH, TocEntry* te) -{ - lclTocEntry* ctx = (lclTocEntry*)te->formatData; - - ahprintf(AH, "-- Data Pos: %d (Length %d)\n", ctx->dataPos, ctx->dataLen); -} - -static void _StartData(ArchiveHandle* AH, TocEntry* te) -{ - lclContext* ctx = (lclContext*)AH->formatData; - z_streamp zp = ctx->zp; - lclTocEntry* tctx = (lclTocEntry*)te->formatData; - - tctx->dataPos = _getFilePos(AH, ctx); - - WriteInt(AH, te->id); /* For sanity check */ - -#ifdef HAVE_LIBZ - - if (AH->compression < 0 || AH->compression > 9) { - AH->compression = Z_DEFAULT_COMPRESSION; - } - - if (AH->compression != 0) { - zp->zalloc = Z_NULL; - zp->zfree = Z_NULL; - zp->opaque = Z_NULL; - - if (deflateInit(zp, AH->compression) != Z_OK) - die_horribly("%s: could not initialize compression library - %s\n",progname, zp->msg); - } - -#else - - AH->compression = 0; - -#endif - - /* Just be paranoid - maye End is called after Start, with no Write */ - zp->next_out = ctx->zlibOut; - zp->avail_out = zlibOutSize; -} - -static int _DoDeflate(ArchiveHandle* AH, lclContext* ctx, int flush) -{ - z_streamp zp = ctx->zp; - -#ifdef HAVE_LIBZ - char* out = ctx->zlibOut; - int res = Z_OK; - - if (AH->compression != 0) - { - res = deflate(zp, flush); - if (res == Z_STREAM_ERROR) - die_horribly("%s: could not compress data - %s\n",progname, zp->msg); - - if ( ( (flush == Z_FINISH) && (zp->avail_out < zlibOutSize) ) - || (zp->avail_out == 0) - || (zp->avail_in != 0) - ) - { - /* - * Extra paranoia: avoid zero-length chunks since a zero - * length chunk is the EOF marker. This should never happen - * but... - */ - if (zp->avail_out < zlibOutSize) { - /* printf("Wrote %d byte deflated chunk\n", zlibOutSize - zp->avail_out); */ - WriteInt(AH, zlibOutSize - zp->avail_out); - fwrite(out, 1, zlibOutSize - zp->avail_out, AH->FH); - ctx->filePos += zlibOutSize - zp->avail_out; - } - zp->next_out = out; - zp->avail_out = zlibOutSize; - } - } else { -#endif - if (zp->avail_in > 0) - { - WriteInt(AH, zp->avail_in); - fwrite(zp->next_in, 1, zp->avail_in, AH->FH); - ctx->filePos += zp->avail_in; - zp->avail_in = 0; - } else { -#ifdef HAVE_LIBZ - if (flush == Z_FINISH) - res = Z_STREAM_END; -#endif - } - - -#ifdef HAVE_LIBZ - } - - return res; -#else - return 1; -#endif - -} - -static int _WriteData(ArchiveHandle* AH, const void* data, int dLen) -{ - lclContext* ctx = (lclContext*)AH->formatData; - z_streamp zp = ctx->zp; - - zp->next_in = (void*)data; - zp->avail_in = dLen; - - while (zp->avail_in != 0) { - /* printf("Deflating %d bytes\n", dLen); */ - _DoDeflate(AH, ctx, 0); - } - return dLen; -} - -static void _EndData(ArchiveHandle* AH, TocEntry* te) -{ - lclContext* ctx = (lclContext*)AH->formatData; - lclTocEntry* tctx = (lclTocEntry*) te->formatData; - -#ifdef HAVE_LIBZ - z_streamp zp = ctx->zp; - int res; - - if (AH->compression != 0) - { - zp->next_in = NULL; - zp->avail_in = 0; - - do { - /* printf("Ending data output\n"); */ - res = _DoDeflate(AH, ctx, Z_FINISH); - } while (res != Z_STREAM_END); - - if (deflateEnd(zp) != Z_OK) - die_horribly("%s: error closing compression stream - %s\n", progname, zp->msg); - } -#endif - - /* Send the end marker */ - WriteInt(AH, 0); - - tctx->dataLen = _getFilePos(AH, ctx) - tctx->dataPos; - -} - -/* - * Print data for a gievn TOC entry -*/ -static void _PrintTocData(ArchiveHandle* AH, TocEntry* te, RestoreOptions *ropt) -{ - lclContext* ctx = (lclContext*)AH->formatData; - int id; - lclTocEntry* tctx = (lclTocEntry*) te->formatData; - - if (tctx->dataPos == 0) - return; - - if (!ctx->hasSeek || tctx->dataPos < 0) { - id = ReadInt(AH); - - while (id != te->id) { - if (TocIDRequired(AH, id, ropt) & 2) - die_horribly("%s: Dumping a specific TOC data block out of order is not supported" - " without on this input stream (fseek required)\n", progname); - _skipData(AH); - id = ReadInt(AH); - } - } else { - - if (fseek(AH->FH, tctx->dataPos, SEEK_SET) != 0) - die_horribly("%s: error %d in file seek\n",progname, errno); - - id = ReadInt(AH); - - } - - if (id != te->id) - die_horribly("%s: Found unexpected block ID (%d) when reading data - expected %d\n", - progname, id, te->id); - - ahprintf(AH, "--\n-- Data for TOC Entry ID %d (OID %s) %s %s\n--\n\n", - te->id, te->oid, te->desc, te->name); - - _PrintData(AH); - - ahprintf(AH, "\n\n"); -} - -/* - * Print data from current file position. -*/ -static void _PrintData(ArchiveHandle* AH) -{ - lclContext* ctx = (lclContext*)AH->formatData; - z_streamp zp = ctx->zp; - int blkLen; - char* in = ctx->zlibIn; - int cnt; - -#ifdef HAVE_LIBZ - - int res; - char* out = ctx->zlibOut; - - res = Z_OK; - - if (AH->compression != 0) { - zp->zalloc = Z_NULL; - zp->zfree = Z_NULL; - zp->opaque = Z_NULL; - - if (inflateInit(zp) != Z_OK) - die_horribly("%s: could not initialize compression library - %s\n", progname, zp->msg); - } - -#endif - - blkLen = ReadInt(AH); - while (blkLen != 0) { - if (blkLen > ctx->inSize) { - free(ctx->zlibIn); - ctx->zlibIn = NULL; - ctx->zlibIn = (char*)malloc(blkLen); - if (!ctx->zlibIn) - die_horribly("%s: failed to allocate decompression buffer\n", progname); - - ctx->inSize = blkLen; - in = ctx->zlibIn; - } - cnt = fread(in, 1, blkLen, AH->FH); - if (cnt != blkLen) - die_horribly("%s: could not read data block - expected %d, got %d\n", progname, blkLen, cnt); - - ctx->filePos += blkLen; - - zp->next_in = in; - zp->avail_in = blkLen; - -#ifdef HAVE_LIBZ - - if (AH->compression != 0) { - - while (zp->avail_in != 0) { - zp->next_out = out; - zp->avail_out = zlibOutSize; - res = inflate(zp, 0); - if (res != Z_OK && res != Z_STREAM_END) - die_horribly("%s: unable to uncompress data - %s\n", progname, zp->msg); - - out[zlibOutSize - zp->avail_out] = '\0'; - ahwrite(out, 1, zlibOutSize - zp->avail_out, AH); - } - } else { -#endif - ahwrite(in, 1, zp->avail_in, AH); - zp->avail_in = 0; - -#ifdef HAVE_LIBZ - } -#endif - - blkLen = ReadInt(AH); - } - -#ifdef HAVE_LIBZ - if (AH->compression != 0) - { - zp->next_in = NULL; - zp->avail_in = 0; - while (res != Z_STREAM_END) { - zp->next_out = out; - zp->avail_out = zlibOutSize; - res = inflate(zp, 0); - if (res != Z_OK && res != Z_STREAM_END) - die_horribly("%s: unable to uncompress data - %s\n", progname, zp->msg); - - out[zlibOutSize - zp->avail_out] = '\0'; - ahwrite(out, 1, zlibOutSize - zp->avail_out, AH); - } - } -#endif - -} - -/* - * Skip data from current file position. -*/ -static void _skipData(ArchiveHandle* AH) -{ - lclContext* ctx = (lclContext*)AH->formatData; - int blkLen; - char* in = ctx->zlibIn; - int cnt; - - blkLen = ReadInt(AH); - while (blkLen != 0) { - if (blkLen > ctx->inSize) { - free(ctx->zlibIn); - ctx->zlibIn = (char*)malloc(blkLen); - ctx->inSize = blkLen; - in = ctx->zlibIn; - } - cnt = fread(in, 1, blkLen, AH->FH); - if (cnt != blkLen) - die_horribly("%s: could not read data block - expected %d, got %d\n", progname, blkLen, cnt); - - ctx->filePos += blkLen; - - blkLen = ReadInt(AH); - } - -} - -static int _WriteByte(ArchiveHandle* AH, const int i) -{ - lclContext* ctx = (lclContext*)AH->formatData; - int res; - - res = fputc(i, AH->FH); - if (res != EOF) { - ctx->filePos += 1; - } - return res; -} - -static int _ReadByte(ArchiveHandle* AH) -{ - lclContext* ctx = (lclContext*)AH->formatData; - int res; - - res = fgetc(AH->FH); - if (res != EOF) { - ctx->filePos += 1; - } - return res; -} - -static int _WriteBuf(ArchiveHandle* AH, const void* buf, int len) -{ - lclContext* ctx = (lclContext*)AH->formatData; - int res; - res = fwrite(buf, 1, len, AH->FH); - ctx->filePos += res; - return res; -} - -static int _ReadBuf(ArchiveHandle* AH, void* buf, int len) -{ - lclContext* ctx = (lclContext*)AH->formatData; - int res; - res = fread(buf, 1, len, AH->FH); - ctx->filePos += res; - return res; -} - -static void _CloseArchive(ArchiveHandle* AH) -{ - lclContext* ctx = (lclContext*)AH->formatData; - int tpos; - - if (AH->mode == archModeWrite) { - WriteHead(AH); - tpos = ftell(AH->FH); - WriteToc(AH); - ctx->dataStart = _getFilePos(AH, ctx); - WriteDataChunks(AH); - /* This is not an essential operation - it is really only - * needed if we expect to be doing seeks to read the data back - * - it may be ok to just use the existing self-consistent block - * formatting. - */ - if (ctx->hasSeek) { - fseek(AH->FH, tpos, SEEK_SET); - WriteToc(AH); - } - } - - fclose(AH->FH); - AH->FH = NULL; -} - -static int _getFilePos(ArchiveHandle* AH, lclContext* ctx) -{ - int pos; - if (ctx->hasSeek) { - pos = ftell(AH->FH); - if (pos != ctx->filePos) { - fprintf(stderr, "Warning: ftell mismatch with filePos\n"); - } - } else { - pos = ctx->filePos; - } - return pos; -} - - +/*------------------------------------------------------------------------- + * + * pg_backup_custom.c + * + * Implements the custom output format. + * + * The comments with the routined in this code are a good place to + * understand how to write a new format. + * + * See the headers to pg_restore for more details. + * + * Copyright (c) 2000, Philip Warner + * Rights are granted to use this software in any way so long + * as this notice is not removed. + * + * The author is not responsible for loss or damages that may + * and any liability will be limited to the time taken to fix any + * related bug. + * + * + * IDENTIFICATION + * + * Modifications - 28-Jun-2000 - pjw@rhyme.com.au + * + * Initial version. + * + *------------------------------------------------------------------------- + */ + +#include +#include "pg_backup.h" +#include "pg_backup_archiver.h" +#include + +/*-------- + * Routines in the format interface + *-------- + */ + +static void _ArchiveEntry(ArchiveHandle* AH, TocEntry* te); +static void _StartData(ArchiveHandle* AH, TocEntry* te); +static int _WriteData(ArchiveHandle* AH, const void* data, int dLen); +static void _EndData(ArchiveHandle* AH, TocEntry* te); +static int _WriteByte(ArchiveHandle* AH, const int i); +static int _ReadByte(ArchiveHandle* ); +static int _WriteBuf(ArchiveHandle* AH, const void* buf, int len); +static int _ReadBuf(ArchiveHandle* AH, void* buf, int len); +static void _CloseArchive(ArchiveHandle* AH); +static void _PrintTocData(ArchiveHandle* AH, TocEntry* te, RestoreOptions *ropt); +static void _WriteExtraToc(ArchiveHandle* AH, TocEntry* te); +static void _ReadExtraToc(ArchiveHandle* AH, TocEntry* te); +static void _PrintExtraToc(ArchiveHandle* AH, TocEntry* te); + +static void _PrintData(ArchiveHandle* AH); +static void _skipData(ArchiveHandle* AH); +static void _skipBlobs(ArchiveHandle* AH); + +static void _StartBlobs(ArchiveHandle* AH, TocEntry* te); +static void _StartBlob(ArchiveHandle* AH, TocEntry* te, int oid); +static void _EndBlob(ArchiveHandle* AH, TocEntry* te, int oid); +static void _EndBlobs(ArchiveHandle* AH, TocEntry* te); +static void _LoadBlobs(ArchiveHandle* AH); + +/*------------ + * Buffers used in zlib compression and extra data stored in archive and + * in TOC entries. + *------------ + */ +#define zlibOutSize 4096 +#define zlibInSize 4096 + +typedef struct { + z_streamp zp; + char* zlibOut; + char* zlibIn; + int inSize; + int hasSeek; + int filePos; + int dataStart; +} lclContext; + +typedef struct { + int dataPos; + int dataLen; +} lclTocEntry; + + +/*------ + * Static declarations + *------ + */ +static void _readBlockHeader(ArchiveHandle *AH, int *type, int *id); +static void _StartDataCompressor(ArchiveHandle* AH, TocEntry* te); +static void _EndDataCompressor(ArchiveHandle* AH, TocEntry* te); +static int _getFilePos(ArchiveHandle* AH, lclContext* ctx); +static int _DoDeflate(ArchiveHandle* AH, lclContext* ctx, int flush); + +static char* progname = "Archiver(custom)"; + + + +/* + * Init routine required by ALL formats. This is a global routine + * and should be declared in pg_backup_archiver.h + * + * It's task is to create any extra archive context (using AH->formatData), + * and to initialize the supported function pointers. + * + * It should also prepare whatever it's input source is for reading/writing, + * and in the case of a read mode connection, it should load the Header & TOC. + */ +void InitArchiveFmt_Custom(ArchiveHandle* AH) +{ + lclContext* ctx; + + /* Assuming static functions, this can be copied for each format. */ + AH->ArchiveEntryPtr = _ArchiveEntry; + AH->StartDataPtr = _StartData; + AH->WriteDataPtr = _WriteData; + AH->EndDataPtr = _EndData; + AH->WriteBytePtr = _WriteByte; + AH->ReadBytePtr = _ReadByte; + AH->WriteBufPtr = _WriteBuf; + AH->ReadBufPtr = _ReadBuf; + AH->ClosePtr = _CloseArchive; + AH->PrintTocDataPtr = _PrintTocData; + AH->ReadExtraTocPtr = _ReadExtraToc; + AH->WriteExtraTocPtr = _WriteExtraToc; + AH->PrintExtraTocPtr = _PrintExtraToc; + + AH->StartBlobsPtr = _StartBlobs; + AH->StartBlobPtr = _StartBlob; + AH->EndBlobPtr = _EndBlob; + AH->EndBlobsPtr = _EndBlobs; + + /* + * Set up some special context used in compressing data. + */ + ctx = (lclContext*)malloc(sizeof(lclContext)); + if (ctx == NULL) + die_horribly(AH, "%s: Unable to allocate archive context",progname); + AH->formatData = (void*)ctx; + + ctx->zp = (z_streamp)malloc(sizeof(z_stream)); + if (ctx->zp == NULL) + die_horribly(AH, "%s: unable to allocate zlib stream archive context",progname); + + ctx->zlibOut = (char*)malloc(zlibOutSize); + ctx->zlibIn = (char*)malloc(zlibInSize); + ctx->inSize = zlibInSize; + ctx->filePos = 0; + + if (ctx->zlibOut == NULL || ctx->zlibIn == NULL) + die_horribly(AH, "%s: unable to allocate buffers in archive context",progname); + + /* + * Now open the file + */ + if (AH->mode == archModeWrite) { + + if (AH->fSpec && strcmp(AH->fSpec,"") != 0) { + AH->FH = fopen(AH->fSpec, PG_BINARY_W); + } else { + AH->FH = stdout; + } + + if (!AH) + die_horribly(AH, "%s: unable to open archive file %s",progname, AH->fSpec); + + ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0); + + } else { + + if (AH->fSpec && strcmp(AH->fSpec,"") != 0) { + AH->FH = fopen(AH->fSpec, PG_BINARY_R); + } else { + AH->FH = stdin; + } + if (!AH) + die_horribly(AH, "%s: unable to open archive file %s",progname, AH->fSpec); + + ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0); + + ReadHead(AH); + ReadToc(AH); + ctx->dataStart = _getFilePos(AH, ctx); + } + +} + +/* + * Called by the Archiver when the dumper creates a new TOC entry. + * + * Optional. + * + * Set up extrac format-related TOC data. +*/ +static void _ArchiveEntry(ArchiveHandle* AH, TocEntry* te) +{ + lclTocEntry* ctx; + + ctx = (lclTocEntry*)calloc(1, sizeof(lclTocEntry)); + if (te->dataDumper) { + ctx->dataPos = -1; + } else { + ctx->dataPos = 0; + } + ctx->dataLen = 0; + te->formatData = (void*)ctx; + +} + +/* + * Called by the Archiver to save any extra format-related TOC entry + * data. + * + * Optional. + * + * Use the Archiver routines to write data - they are non-endian, and + * maintain other important file information. + */ +static void _WriteExtraToc(ArchiveHandle* AH, TocEntry* te) +{ + lclTocEntry* ctx = (lclTocEntry*)te->formatData; + + WriteInt(AH, ctx->dataPos); + WriteInt(AH, ctx->dataLen); +} + +/* + * Called by the Archiver to read any extra format-related TOC data. + * + * Optional. + * + * Needs to match the order defined in _WriteExtraToc, and sould also + * use the Archiver input routines. + */ +static void _ReadExtraToc(ArchiveHandle* AH, TocEntry* te) +{ + lclTocEntry* ctx = (lclTocEntry*)te->formatData; + + if (ctx == NULL) { + ctx = (lclTocEntry*)malloc(sizeof(lclTocEntry)); + te->formatData = (void*)ctx; + } + + ctx->dataPos = ReadInt( AH ); + ctx->dataLen = ReadInt( AH ); +} + +/* + * Called by the Archiver when restoring an archive to output a comment + * that includes useful information about the TOC entry. + * + * Optional. + * + */ +static void _PrintExtraToc(ArchiveHandle* AH, TocEntry* te) +{ + lclTocEntry* ctx = (lclTocEntry*)te->formatData; + + ahprintf(AH, "-- Data Pos: %d (Length %d)\n", ctx->dataPos, ctx->dataLen); +} + +/* + * Called by the archiver when saving TABLE DATA (not schema). This routine + * should save whatever format-specific information is needed to read + * the archive back. + * + * It is called just prior to the dumper's 'DataDumper' routine being called. + * + * Optional, but strongly recommended. + * + */ +static void _StartData(ArchiveHandle* AH, TocEntry* te) +{ + lclContext* ctx = (lclContext*)AH->formatData; + lclTocEntry* tctx = (lclTocEntry*)te->formatData; + + tctx->dataPos = _getFilePos(AH, ctx); + + _WriteByte(AH, BLK_DATA); /* Block type */ + WriteInt(AH, te->id); /* For sanity check */ + + _StartDataCompressor(AH, te); + +} + +/* + * Called by archiver when dumper calls WriteData. This routine is + * called for both BLOB and TABLE data; it is the responsibility of + * the format to manage each kind of data using StartBlob/StartData. + * + * It should only be called from withing a DataDumper routine. + * + * Mandatory. + * + */ +static int _WriteData(ArchiveHandle* AH, const void* data, int dLen) +{ + lclContext* ctx = (lclContext*)AH->formatData; + z_streamp zp = ctx->zp; + + zp->next_in = (void*)data; + zp->avail_in = dLen; + + while (zp->avail_in != 0) { + /* printf("Deflating %d bytes\n", dLen); */ + _DoDeflate(AH, ctx, 0); + } + return dLen; +} + +/* + * Called by the archiver when a dumper's 'DataDumper' routine has + * finished. + * + * Optional. + * + */ +static void _EndData(ArchiveHandle* AH, TocEntry* te) +{ + lclContext* ctx = (lclContext*)AH->formatData; + lclTocEntry* tctx = (lclTocEntry*) te->formatData; + + _EndDataCompressor(AH, te); + + tctx->dataLen = _getFilePos(AH, ctx) - tctx->dataPos; +} + +/* + * Called by the archiver when starting to save all BLOB DATA (not schema). + * This routine should save whatever format-specific information is needed + * to read the BLOBs back into memory. + * + * It is called just prior to the dumper's DataDumper routine. + * + * Optional, but strongly recommended. + * + */ +static void _StartBlobs(ArchiveHandle* AH, TocEntry* te) +{ + lclContext* ctx = (lclContext*)AH->formatData; + lclTocEntry* tctx = (lclTocEntry*)te->formatData; + + tctx->dataPos = _getFilePos(AH, ctx); + + _WriteByte(AH, BLK_BLOBS); /* Block type */ + WriteInt(AH, te->id); /* For sanity check */ + +} + +/* + * Called by the archiver when the dumper calls StartBlob. + * + * Mandatory. + * + * Must save the passed OID for retrieval at restore-time. + */ +static void _StartBlob(ArchiveHandle* AH, TocEntry* te, int oid) +{ + if (oid == 0) + die_horribly(AH, "%s: illegal OID for BLOB (%d)\n", progname, oid); + + WriteInt(AH, oid); + _StartDataCompressor(AH, te); +} + +/* + * Called by the archiver when the dumper calls EndBlob. + * + * Optional. + * + */ +static void _EndBlob(ArchiveHandle* AH, TocEntry* te, int oid) +{ + _EndDataCompressor(AH, te); +} + +/* + * Called by the archiver when finishing saving all BLOB DATA. + * + * Optional. + * + */ +static void _EndBlobs(ArchiveHandle* AH, TocEntry* te) +{ + /* Write out a fake zero OID to mark end-of-blobs. */ + WriteInt(AH, 0); +} + +/* + * Print data for a gievn TOC entry +*/ +static void _PrintTocData(ArchiveHandle* AH, TocEntry* te, RestoreOptions *ropt) +{ + lclContext* ctx = (lclContext*)AH->formatData; + int id; + lclTocEntry* tctx = (lclTocEntry*) te->formatData; + int blkType; + int found = 0; + + if (tctx->dataPos == 0) + return; + + if (!ctx->hasSeek || tctx->dataPos < 0) { + + /* Skip over unnecessary blocks until we get the one we want. */ + + found = 0; + + _readBlockHeader(AH, &blkType, &id); + + while (id != te->id) { + + if ( (TocIDRequired(AH, id, ropt) & 2) != 0) + die_horribly(AH, "%s: Dumping a specific TOC data block out of order is not supported" + " without on this input stream (fseek required)\n", progname); + + switch (blkType) { + + case BLK_DATA: + + _skipData(AH); + break; + + case BLK_BLOBS: + + _skipBlobs(AH); + break; + + default: /* Always have a default */ + + die_horribly(AH, "%s: unrecognized data block type while searching archive %d\n", + progname, blkType); + break; + } + + _readBlockHeader(AH, &blkType, &id); + + } + + } else { + + /* Grab it */ + + if (fseek(AH->FH, tctx->dataPos, SEEK_SET) != 0) + die_horribly(AH, "%s: error %d in file seek\n",progname, errno); + + _readBlockHeader(AH, &blkType, &id); + + } + + /* Are we sane? */ + if (id != te->id) + die_horribly(AH, "%s: Found unexpected block ID (%d) when reading data - expected %d\n", + progname, id, te->id); + + switch (blkType) { + + case BLK_DATA: + + _PrintData(AH); + break; + + case BLK_BLOBS: + + if (!AH->connection) + die_horribly(AH, "%s: BLOBs can not be loaded without a database connection\n", progname); + + _LoadBlobs(AH); + break; + + default: /* Always have a default */ + + die_horribly(AH, "%s: unrecognized data block type %d while restoring archive\n", + progname, blkType); + break; + } + + ahprintf(AH, "\n\n"); +} + +/* + * Print data from current file position. +*/ +static void _PrintData(ArchiveHandle* AH) +{ + lclContext* ctx = (lclContext*)AH->formatData; + z_streamp zp = ctx->zp; + int blkLen; + char* in = ctx->zlibIn; + int cnt; +#ifdef HAVE_LIBZ + int res; + char* out = ctx->zlibOut; +#endif + +#ifdef HAVE_LIBZ + + res = Z_OK; + + if (AH->compression != 0) { + zp->zalloc = Z_NULL; + zp->zfree = Z_NULL; + zp->opaque = Z_NULL; + + if (inflateInit(zp) != Z_OK) + die_horribly(AH, "%s: could not initialize compression library - %s\n", progname, zp->msg); + } + +#endif + + blkLen = ReadInt(AH); + while (blkLen != 0) { + if (blkLen > (ctx->inSize - 1)) { + free(ctx->zlibIn); + ctx->zlibIn = NULL; + ctx->zlibIn = (char*)malloc(blkLen); + if (!ctx->zlibIn) + die_horribly(AH, "%s: failed to allocate decompression buffer\n", progname); + + ctx->inSize = blkLen; + in = ctx->zlibIn; + } + + cnt = fread(in, 1, blkLen, AH->FH); + if (cnt != blkLen) + die_horribly(AH, "%s: could not read data block - expected %d, got %d\n", progname, blkLen, cnt); + + ctx->filePos += blkLen; + + zp->next_in = in; + zp->avail_in = blkLen; + +#ifdef HAVE_LIBZ + + if (AH->compression != 0) { + + while (zp->avail_in != 0) { + zp->next_out = out; + zp->avail_out = zlibOutSize; + res = inflate(zp, 0); + if (res != Z_OK && res != Z_STREAM_END) + die_horribly(AH, "%s: unable to uncompress data - %s\n", progname, zp->msg); + + out[zlibOutSize - zp->avail_out] = '\0'; + ahwrite(out, 1, zlibOutSize - zp->avail_out, AH); + } + } else { +#endif + in[zp->avail_in] = '\0'; + ahwrite(in, 1, zp->avail_in, AH); + zp->avail_in = 0; + +#ifdef HAVE_LIBZ + } +#endif + + blkLen = ReadInt(AH); + + } + +#ifdef HAVE_LIBZ + if (AH->compression != 0) + { + zp->next_in = NULL; + zp->avail_in = 0; + while (res != Z_STREAM_END) { + zp->next_out = out; + zp->avail_out = zlibOutSize; + res = inflate(zp, 0); + if (res != Z_OK && res != Z_STREAM_END) + die_horribly(AH, "%s: unable to uncompress data - %s\n", progname, zp->msg); + + out[zlibOutSize - zp->avail_out] = '\0'; + ahwrite(out, 1, zlibOutSize - zp->avail_out, AH); + } + } +#endif + +} + +static void _LoadBlobs(ArchiveHandle* AH) +{ + int oid; + + oid = ReadInt(AH); + while(oid != 0) + { + StartRestoreBlob(AH, oid); + _PrintData(AH); + EndRestoreBlob(AH, oid); + oid = ReadInt(AH); + } +} + +/* + * Skip the BLOBs from the current file position. + * BLOBS are written sequentially as data blocks (see below). + * Each BLOB is preceded by it's original OID. + * A zero OID indicated the end of the BLOBS + */ +static void _skipBlobs(ArchiveHandle* AH) +{ + int oid; + + oid = ReadInt(AH); + while(oid != 0) + { + _skipData(AH); + oid = ReadInt(AH); + } +} + +/* + * Skip data from current file position. + * Data blocks are formatted as an integer length, followed by data. + * A zero length denoted the end of the block. +*/ +static void _skipData(ArchiveHandle* AH) +{ + lclContext* ctx = (lclContext*)AH->formatData; + int blkLen; + char* in = ctx->zlibIn; + int cnt; + + blkLen = ReadInt(AH); + while (blkLen != 0) { + if (blkLen > ctx->inSize) { + free(ctx->zlibIn); + ctx->zlibIn = (char*)malloc(blkLen); + ctx->inSize = blkLen; + in = ctx->zlibIn; + } + cnt = fread(in, 1, blkLen, AH->FH); + if (cnt != blkLen) + die_horribly(AH, "%s: could not read data block - expected %d, got %d\n", progname, blkLen, cnt); + + ctx->filePos += blkLen; + + blkLen = ReadInt(AH); + } + +} + +/* + * Write a byte of data to the archive. + * + * Mandatory. + * + * Called by the archiver to do integer & byte output to the archive. + * These routines are only used to read & write headers & TOC. + * + */ +static int _WriteByte(ArchiveHandle* AH, const int i) +{ + lclContext* ctx = (lclContext*)AH->formatData; + int res; + + res = fputc(i, AH->FH); + if (res != EOF) { + ctx->filePos += 1; + } + return res; +} + +/* + * Read a byte of data from the archive. + * + * Mandatory + * + * Called by the archiver to read bytes & integers from the archive. + * These routines are only used to read & write headers & TOC. + * + */ +static int _ReadByte(ArchiveHandle* AH) +{ + lclContext* ctx = (lclContext*)AH->formatData; + int res; + + res = fgetc(AH->FH); + if (res != EOF) { + ctx->filePos += 1; + } + return res; +} + +/* + * Write a buffer of data to the archive. + * + * Mandatory. + * + * Called by the archiver to write a block of bytes to the archive. + * These routines are only used to read & write headers & TOC. + * + */ +static int _WriteBuf(ArchiveHandle* AH, const void* buf, int len) +{ + lclContext* ctx = (lclContext*)AH->formatData; + int res; + res = fwrite(buf, 1, len, AH->FH); + ctx->filePos += res; + return res; +} + +/* + * Read a block of bytes from the archive. + * + * Mandatory. + * + * Called by the archiver to read a block of bytes from the archive + * These routines are only used to read & write headers & TOC. + * + */ +static int _ReadBuf(ArchiveHandle* AH, void* buf, int len) +{ + lclContext* ctx = (lclContext*)AH->formatData; + int res; + res = fread(buf, 1, len, AH->FH); + ctx->filePos += res; + return res; +} + +/* + * Close the archive. + * + * Mandatory. + * + * When writing the archive, this is the routine that actually starts + * the process of saving it to files. No data should be written prior + * to this point, since the user could sort the TOC after creating it. + * + * If an archive is to be written, this toutine must call: + * WriteHead to save the archive header + * WriteToc to save the TOC entries + * WriteDataChunks to save all DATA & BLOBs. + * + */ +static void _CloseArchive(ArchiveHandle* AH) +{ + lclContext* ctx = (lclContext*)AH->formatData; + int tpos; + + if (AH->mode == archModeWrite) { + WriteHead(AH); + tpos = ftell(AH->FH); + WriteToc(AH); + ctx->dataStart = _getFilePos(AH, ctx); + WriteDataChunks(AH); + /* This is not an essential operation - it is really only + * needed if we expect to be doing seeks to read the data back + * - it may be ok to just use the existing self-consistent block + * formatting. + */ + if (ctx->hasSeek) { + fseek(AH->FH, tpos, SEEK_SET); + WriteToc(AH); + } + } + + fclose(AH->FH); + AH->FH = NULL; +} + +/*-------------------------------------------------- + * END OF FORMAT CALLBACKS + *-------------------------------------------------- + */ + +/* + * Get the current position in the archive file. + */ +static int _getFilePos(ArchiveHandle* AH, lclContext* ctx) +{ + int pos; + if (ctx->hasSeek) { + pos = ftell(AH->FH); + if (pos != ctx->filePos) { + fprintf(stderr, "Warning: ftell mismatch with filePos\n"); + } + } else { + pos = ctx->filePos; + } + return pos; +} + +/* + * Read a data block header. The format changed in V1.3, so we + * put the code here for simplicity. + */ +static void _readBlockHeader(ArchiveHandle *AH, int *type, int *id) +{ + if (AH->version < K_VERS_1_3) + *type = BLK_DATA; + else + *type = _ReadByte(AH);; + + *id = ReadInt(AH); +} + +/* + * If zlib is available, then startit up. This is called from + * StartData & StartBlob. The buffers are setup in the Init routine. + * + */ +static void _StartDataCompressor(ArchiveHandle* AH, TocEntry* te) +{ + lclContext* ctx = (lclContext*)AH->formatData; + z_streamp zp = ctx->zp; + +#ifdef HAVE_LIBZ + + if (AH->compression < 0 || AH->compression > 9) { + AH->compression = Z_DEFAULT_COMPRESSION; + } + + if (AH->compression != 0) { + zp->zalloc = Z_NULL; + zp->zfree = Z_NULL; + zp->opaque = Z_NULL; + + if (deflateInit(zp, AH->compression) != Z_OK) + die_horribly(AH, "%s: could not initialize compression library - %s\n",progname, zp->msg); + } + +#else + + AH->compression = 0; + +#endif + + /* Just be paranoid - maye End is called after Start, with no Write */ + zp->next_out = ctx->zlibOut; + zp->avail_out = zlibOutSize; +} + +/* + * Send compressed data to the output stream (via ahwrite). + * Each data chunk is preceded by it's length. + * In the case of Z0, or no zlib, just write the raw data. + * + */ +static int _DoDeflate(ArchiveHandle* AH, lclContext* ctx, int flush) +{ + z_streamp zp = ctx->zp; + +#ifdef HAVE_LIBZ + char* out = ctx->zlibOut; + int res = Z_OK; + + if (AH->compression != 0) + { + res = deflate(zp, flush); + if (res == Z_STREAM_ERROR) + die_horribly(AH, "%s: could not compress data - %s\n",progname, zp->msg); + + if ( ( (flush == Z_FINISH) && (zp->avail_out < zlibOutSize) ) + || (zp->avail_out == 0) + || (zp->avail_in != 0) + ) + { + /* + * Extra paranoia: avoid zero-length chunks since a zero + * length chunk is the EOF marker. This should never happen + * but... + */ + if (zp->avail_out < zlibOutSize) { + /* printf("Wrote %d byte deflated chunk\n", zlibOutSize - zp->avail_out); */ + WriteInt(AH, zlibOutSize - zp->avail_out); + fwrite(out, 1, zlibOutSize - zp->avail_out, AH->FH); + ctx->filePos += zlibOutSize - zp->avail_out; + } + zp->next_out = out; + zp->avail_out = zlibOutSize; + } + } else { +#endif + if (zp->avail_in > 0) + { + WriteInt(AH, zp->avail_in); + fwrite(zp->next_in, 1, zp->avail_in, AH->FH); + ctx->filePos += zp->avail_in; + zp->avail_in = 0; + } else { +#ifdef HAVE_LIBZ + if (flush == Z_FINISH) + res = Z_STREAM_END; +#endif + } + + +#ifdef HAVE_LIBZ + } + + return res; +#else + return 1; +#endif + +} + +/* + * Terminate zlib context and flush it's buffers. If no zlib + * then just return. + * + */ +static void _EndDataCompressor(ArchiveHandle* AH, TocEntry* te) +{ + +#ifdef HAVE_LIBZ + lclContext* ctx = (lclContext*)AH->formatData; + z_streamp zp = ctx->zp; + int res; + + if (AH->compression != 0) + { + zp->next_in = NULL; + zp->avail_in = 0; + + do { + /* printf("Ending data output\n"); */ + res = _DoDeflate(AH, ctx, Z_FINISH); + } while (res != Z_STREAM_END); + + if (deflateEnd(zp) != Z_OK) + die_horribly(AH, "%s: error closing compression stream - %s\n", progname, zp->msg); + } +#endif + + /* Send the end marker */ + WriteInt(AH, 0); +} + + diff --git a/src/bin/pg_dump/pg_backup_files.c b/src/bin/pg_dump/pg_backup_files.c index ef2ea57f2c..1583a497b9 100644 --- a/src/bin/pg_dump/pg_backup_files.c +++ b/src/bin/pg_dump/pg_backup_files.c @@ -1,303 +1,483 @@ -/*------------------------------------------------------------------------- - * - * pg_backup_files.c - * - * This file is copied from the 'custom' format file, but dumps data into - * separate files, and the TOC into the 'main' file. - * - * IT IS FOR DEMONSTRATION PURPOSES ONLY. - * - * (and could probably be used as a basis for writing a tar file) - * - * See the headers to pg_restore for more details. - * - * Copyright (c) 2000, Philip Warner - * Rights are granted to use this software in any way so long - * as this notice is not removed. - * - * The author is not responsible for loss or damages that may - * result from it's use. - * - * - * IDENTIFICATION - * - * Modifications - 28-Jun-2000 - pjw@rhyme.com.au - * - * Initial version. - * - *------------------------------------------------------------------------- - */ - -#include -#include -#include "pg_backup.h" -#include "pg_backup_archiver.h" - -static void _ArchiveEntry(ArchiveHandle* AH, TocEntry* te); -static void _StartData(ArchiveHandle* AH, TocEntry* te); -static int _WriteData(ArchiveHandle* AH, const void* data, int dLen); -static void _EndData(ArchiveHandle* AH, TocEntry* te); -static int _WriteByte(ArchiveHandle* AH, const int i); -static int _ReadByte(ArchiveHandle* ); -static int _WriteBuf(ArchiveHandle* AH, const void* buf, int len); -static int _ReadBuf(ArchiveHandle* AH, void* buf, int len); -static void _CloseArchive(ArchiveHandle* AH); -static void _PrintTocData(ArchiveHandle* AH, TocEntry* te, RestoreOptions *ropt); -static void _WriteExtraToc(ArchiveHandle* AH, TocEntry* te); -static void _ReadExtraToc(ArchiveHandle* AH, TocEntry* te); -static void _PrintExtraToc(ArchiveHandle* AH, TocEntry* te); - - -typedef struct { - int hasSeek; - int filePos; -} lclContext; - -typedef struct { -#ifdef HAVE_LIBZ - gzFile *FH; -#else - FILE *FH; -#endif - char *filename; -} lclTocEntry; - -/* - * Initializer - */ -void InitArchiveFmt_Files(ArchiveHandle* AH) -{ - lclContext* ctx; - - /* Assuming static functions, this can be copied for each format. */ - AH->ArchiveEntryPtr = _ArchiveEntry; - AH->StartDataPtr = _StartData; - AH->WriteDataPtr = _WriteData; - AH->EndDataPtr = _EndData; - AH->WriteBytePtr = _WriteByte; - AH->ReadBytePtr = _ReadByte; - AH->WriteBufPtr = _WriteBuf; - AH->ReadBufPtr = _ReadBuf; - AH->ClosePtr = _CloseArchive; - AH->PrintTocDataPtr = _PrintTocData; - AH->ReadExtraTocPtr = _ReadExtraToc; - AH->WriteExtraTocPtr = _WriteExtraToc; - AH->PrintExtraTocPtr = _PrintExtraToc; - - /* - * Set up some special context used in compressing data. - */ - ctx = (lclContext*)malloc(sizeof(lclContext)); - AH->formatData = (void*)ctx; - ctx->filePos = 0; - - /* - * Now open the TOC file - */ - if (AH->mode == archModeWrite) { - if (AH->fSpec && strcmp(AH->fSpec,"") != 0) { - AH->FH = fopen(AH->fSpec, PG_BINARY_W); - } else { - AH->FH = stdout; - } - ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0); - - if (AH->compression < 0 || AH->compression > 9) { - AH->compression = Z_DEFAULT_COMPRESSION; - } - - - } else { - if (AH->fSpec && strcmp(AH->fSpec,"") != 0) { - AH->FH = fopen(AH->fSpec, PG_BINARY_R); - } else { - AH->FH = stdin; - } - ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0); - - ReadHead(AH); - ReadToc(AH); - fclose(AH->FH); /* Nothing else in the file... */ - } - -} - -/* - * - Start a new TOC entry - * Setup the output file name. - */ -static void _ArchiveEntry(ArchiveHandle* AH, TocEntry* te) -{ - lclTocEntry* ctx; - char fn[1024]; - - ctx = (lclTocEntry*)malloc(sizeof(lclTocEntry)); - if (te->dataDumper) { -#ifdef HAVE_LIBZ - if (AH->compression == 0) { - sprintf(fn, "%d.dat", te->id); - } else { - sprintf(fn, "%d.dat.gz", te->id); - } -#else - sprintf(fn, "%d.dat", te->id); -#endif - ctx->filename = strdup(fn); - } else { - ctx->filename = NULL; - ctx->FH = NULL; - } - te->formatData = (void*)ctx; -} - -static void _WriteExtraToc(ArchiveHandle* AH, TocEntry* te) -{ - lclTocEntry* ctx = (lclTocEntry*)te->formatData; - - if (ctx->filename) { - WriteStr(AH, ctx->filename); - } else { - WriteStr(AH, ""); - } -} - -static void _ReadExtraToc(ArchiveHandle* AH, TocEntry* te) -{ - lclTocEntry* ctx = (lclTocEntry*)te->formatData; - - if (ctx == NULL) { - ctx = (lclTocEntry*)malloc(sizeof(lclTocEntry)); - te->formatData = (void*)ctx; - } - - ctx->filename = ReadStr(AH); - if (strlen(ctx->filename) == 0) { - free(ctx->filename); - ctx->filename = NULL; - } - ctx->FH = NULL; -} - -static void _PrintExtraToc(ArchiveHandle* AH, TocEntry* te) -{ - lclTocEntry* ctx = (lclTocEntry*)te->formatData; - - ahprintf(AH, "-- File: %s\n", ctx->filename); -} - -static void _StartData(ArchiveHandle* AH, TocEntry* te) -{ - lclTocEntry* tctx = (lclTocEntry*)te->formatData; - char fmode[10]; - - sprintf(fmode, "wb%d", AH->compression); - -#ifdef HAVE_LIBZ - tctx->FH = gzopen(tctx->filename, fmode); -#else - tctx->FH = fopen(tctx->filename, PG_BINARY_W); -#endif -} - -static int _WriteData(ArchiveHandle* AH, const void* data, int dLen) -{ - lclTocEntry* tctx = (lclTocEntry*)AH->currToc->formatData; - - GZWRITE((void*)data, 1, dLen, tctx->FH); - - return dLen; -} - -static void _EndData(ArchiveHandle* AH, TocEntry* te) -{ - lclTocEntry* tctx = (lclTocEntry*) te->formatData; - - /* Close the file */ - GZCLOSE(tctx->FH); - tctx->FH = NULL; -} - -/* - * Print data for a given TOC entry -*/ -static void _PrintTocData(ArchiveHandle* AH, TocEntry* te, RestoreOptions *ropt) -{ - lclTocEntry* tctx = (lclTocEntry*) te->formatData; - char buf[4096]; - int cnt; - - if (!tctx->filename) - return; - -#ifdef HAVE_LIBZ - AH->FH = gzopen(tctx->filename,"rb"); -#else - AH->FH = fopen(tctx->filename,PG_BINARY_R); -#endif - - ahprintf(AH, "--\n-- Data for TOC Entry ID %d (OID %s) %s %s\n--\n\n", - te->id, te->oid, te->desc, te->name); - - while ( (cnt = GZREAD(buf, 1, 4096, AH->FH)) > 0) { - ahwrite(buf, 1, cnt, AH); - } - - GZCLOSE(AH->FH); - - ahprintf(AH, "\n\n"); -} - -static int _WriteByte(ArchiveHandle* AH, const int i) -{ - lclContext* ctx = (lclContext*)AH->formatData; - int res; - - res = fputc(i, AH->FH); - if (res != EOF) { - ctx->filePos += 1; - } - return res; -} - -static int _ReadByte(ArchiveHandle* AH) -{ - lclContext* ctx = (lclContext*)AH->formatData; - int res; - - res = fgetc(AH->FH); - if (res != EOF) { - ctx->filePos += 1; - } - return res; -} - -static int _WriteBuf(ArchiveHandle* AH, const void* buf, int len) -{ - lclContext* ctx = (lclContext*)AH->formatData; - int res; - res = fwrite(buf, 1, len, AH->FH); - ctx->filePos += res; - return res; -} - -static int _ReadBuf(ArchiveHandle* AH, void* buf, int len) -{ - lclContext* ctx = (lclContext*)AH->formatData; - int res; - res = fread(buf, 1, len, AH->FH); - ctx->filePos += res; - return res; -} - -static void _CloseArchive(ArchiveHandle* AH) -{ - if (AH->mode == archModeWrite) { - WriteHead(AH); - WriteToc(AH); - fclose(AH->FH); - WriteDataChunks(AH); - } - - AH->FH = NULL; -} - +/*------------------------------------------------------------------------- + * + * pg_backup_files.c + * + * This file is copied from the 'custom' format file, but dumps data into + * separate files, and the TOC into the 'main' file. + * + * IT IS FOR DEMONSTRATION PURPOSES ONLY. + * + * (and could probably be used as a basis for writing a tar file) + * + * See the headers to pg_restore for more details. + * + * Copyright (c) 2000, Philip Warner + * Rights are granted to use this software in any way so long + * as this notice is not removed. + * + * The author is not responsible for loss or damages that may + * result from it's use. + * + * + * IDENTIFICATION + * + * Modifications - 28-Jun-2000 - pjw@rhyme.com.au + * + * Initial version. + * + *------------------------------------------------------------------------- + */ + +#include +#include +#include "pg_backup.h" +#include "pg_backup_archiver.h" + +static void _ArchiveEntry(ArchiveHandle* AH, TocEntry* te); +static void _StartData(ArchiveHandle* AH, TocEntry* te); +static int _WriteData(ArchiveHandle* AH, const void* data, int dLen); +static void _EndData(ArchiveHandle* AH, TocEntry* te); +static int _WriteByte(ArchiveHandle* AH, const int i); +static int _ReadByte(ArchiveHandle* ); +static int _WriteBuf(ArchiveHandle* AH, const void* buf, int len); +static int _ReadBuf(ArchiveHandle* AH, void* buf, int len); +static void _CloseArchive(ArchiveHandle* AH); +static void _PrintTocData(ArchiveHandle* AH, TocEntry* te, RestoreOptions *ropt); +static void _WriteExtraToc(ArchiveHandle* AH, TocEntry* te); +static void _ReadExtraToc(ArchiveHandle* AH, TocEntry* te); +static void _PrintExtraToc(ArchiveHandle* AH, TocEntry* te); + +static void _StartBlobs(ArchiveHandle* AH, TocEntry* te); +static void _StartBlob(ArchiveHandle* AH, TocEntry* te, int oid); +static void _EndBlob(ArchiveHandle* AH, TocEntry* te, int oid); +static void _EndBlobs(ArchiveHandle* AH, TocEntry* te); + +#define K_STD_BUF_SIZE 1024 + +typedef struct { + int hasSeek; + int filePos; + FILE *blobToc; +} lclContext; + +typedef struct { +#ifdef HAVE_LIBZ + gzFile *FH; +#else + FILE *FH; +#endif + char *filename; +} lclTocEntry; + +static char* progname = "Archiver(files)"; +static void _LoadBlobs(ArchiveHandle* AH, RestoreOptions *ropt); +static void _getBlobTocEntry(ArchiveHandle* AH, int *oid, char *fname); + +/* + * Initializer + */ +void InitArchiveFmt_Files(ArchiveHandle* AH) +{ + lclContext* ctx; + + /* Assuming static functions, this can be copied for each format. */ + AH->ArchiveEntryPtr = _ArchiveEntry; + AH->StartDataPtr = _StartData; + AH->WriteDataPtr = _WriteData; + AH->EndDataPtr = _EndData; + AH->WriteBytePtr = _WriteByte; + AH->ReadBytePtr = _ReadByte; + AH->WriteBufPtr = _WriteBuf; + AH->ReadBufPtr = _ReadBuf; + AH->ClosePtr = _CloseArchive; + AH->PrintTocDataPtr = _PrintTocData; + AH->ReadExtraTocPtr = _ReadExtraToc; + AH->WriteExtraTocPtr = _WriteExtraToc; + AH->PrintExtraTocPtr = _PrintExtraToc; + + AH->StartBlobsPtr = _StartBlobs; + AH->StartBlobPtr = _StartBlob; + AH->EndBlobPtr = _EndBlob; + AH->EndBlobsPtr = _EndBlobs; + + /* + * Set up some special context used in compressing data. + */ + ctx = (lclContext*)malloc(sizeof(lclContext)); + AH->formatData = (void*)ctx; + ctx->filePos = 0; + + /* + * Now open the TOC file + */ + if (AH->mode == archModeWrite) { + + fprintf(stderr, "\n*************************************************************\n" + "* WARNING: This format is for demonstration purposes. It is *\n" + "* not intended for general use. Files will be dumped *\n" + "* into the current working directory. *\n" + "***************************************************************\n\n"); + + if (AH->fSpec && strcmp(AH->fSpec,"") != 0) { + AH->FH = fopen(AH->fSpec, PG_BINARY_W); + } else { + AH->FH = stdout; + } + ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0); + + if (AH->compression < 0 || AH->compression > 9) { + AH->compression = Z_DEFAULT_COMPRESSION; + } + + + } else { /* Read Mode */ + + if (AH->fSpec && strcmp(AH->fSpec,"") != 0) { + AH->FH = fopen(AH->fSpec, PG_BINARY_R); + } else { + AH->FH = stdin; + } + ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0); + + ReadHead(AH); + ReadToc(AH); + fclose(AH->FH); /* Nothing else in the file... */ + } + +} + +/* + * - Start a new TOC entry + * Setup the output file name. + */ +static void _ArchiveEntry(ArchiveHandle* AH, TocEntry* te) +{ + lclTocEntry* ctx; + char fn[K_STD_BUF_SIZE]; + + ctx = (lclTocEntry*)malloc(sizeof(lclTocEntry)); + if (te->dataDumper) { +#ifdef HAVE_LIBZ + if (AH->compression == 0) { + sprintf(fn, "%d.dat", te->id); + } else { + sprintf(fn, "%d.dat.gz", te->id); + } +#else + sprintf(fn, "%d.dat", te->id); +#endif + ctx->filename = strdup(fn); + } else { + ctx->filename = NULL; + ctx->FH = NULL; + } + te->formatData = (void*)ctx; +} + +static void _WriteExtraToc(ArchiveHandle* AH, TocEntry* te) +{ + lclTocEntry* ctx = (lclTocEntry*)te->formatData; + + if (ctx->filename) { + WriteStr(AH, ctx->filename); + } else { + WriteStr(AH, ""); + } +} + +static void _ReadExtraToc(ArchiveHandle* AH, TocEntry* te) +{ + lclTocEntry* ctx = (lclTocEntry*)te->formatData; + + if (ctx == NULL) { + ctx = (lclTocEntry*)malloc(sizeof(lclTocEntry)); + te->formatData = (void*)ctx; + } + + ctx->filename = ReadStr(AH); + if (strlen(ctx->filename) == 0) { + free(ctx->filename); + ctx->filename = NULL; + } + ctx->FH = NULL; +} + +static void _PrintExtraToc(ArchiveHandle* AH, TocEntry* te) +{ + lclTocEntry* ctx = (lclTocEntry*)te->formatData; + + ahprintf(AH, "-- File: %s\n", ctx->filename); +} + +static void _StartData(ArchiveHandle* AH, TocEntry* te) +{ + lclTocEntry* tctx = (lclTocEntry*)te->formatData; + char fmode[10]; + + sprintf(fmode, "wb%d", AH->compression); + +#ifdef HAVE_LIBZ + tctx->FH = gzopen(tctx->filename, fmode); +#else + tctx->FH = fopen(tctx->filename, PG_BINARY_W); +#endif +} + +static int _WriteData(ArchiveHandle* AH, const void* data, int dLen) +{ + lclTocEntry* tctx = (lclTocEntry*)AH->currToc->formatData; + + GZWRITE((void*)data, 1, dLen, tctx->FH); + + return dLen; +} + +static void _EndData(ArchiveHandle* AH, TocEntry* te) +{ + lclTocEntry* tctx = (lclTocEntry*) te->formatData; + + /* Close the file */ + GZCLOSE(tctx->FH); + tctx->FH = NULL; +} + +/* + * Print data for a given file + */ +static void _PrintFileData(ArchiveHandle* AH, char *filename, RestoreOptions *ropt) +{ + char buf[4096]; + int cnt; + + if (!filename) + return; + +#ifdef HAVE_LIBZ + AH->FH = gzopen(filename,"rb"); +#else + AH->FH = fopen(filename,PG_BINARY_R); +#endif + + while ( (cnt = GZREAD(buf, 1, 4095, AH->FH)) > 0) { + buf[cnt] = '\0'; + ahwrite(buf, 1, cnt, AH); + } + + GZCLOSE(AH->FH); +} + + +/* + * Print data for a given TOC entry +*/ +static void _PrintTocData(ArchiveHandle* AH, TocEntry* te, RestoreOptions *ropt) +{ + lclTocEntry* tctx = (lclTocEntry*) te->formatData; + + if (!tctx->filename) + return; + + if (strcmp(te->desc, "BLOBS") == 0) + _LoadBlobs(AH, ropt); + else + { + _PrintFileData(AH, tctx->filename, ropt); + } +} + +static void _getBlobTocEntry(ArchiveHandle* AH, int *oid, char fname[K_STD_BUF_SIZE]) +{ + lclContext* ctx = (lclContext*)AH->formatData; + char blobTe[K_STD_BUF_SIZE]; + int fpos; + int eos; + + if (fgets(&blobTe[0], K_STD_BUF_SIZE - 1, ctx->blobToc) != NULL) + { + *oid = atoi(blobTe); + + fpos = strcspn(blobTe, " "); + + strncpy(fname, &blobTe[fpos+1], K_STD_BUF_SIZE - 1); + + eos = strlen(fname)-1; + + if (fname[eos] == '\n') + fname[eos] = '\0'; + + } else { + + *oid = 0; + fname[0] = '\0'; + } +} + +static void _LoadBlobs(ArchiveHandle* AH, RestoreOptions *ropt) +{ + int oid; + lclContext* ctx = (lclContext*)AH->formatData; + char fname[K_STD_BUF_SIZE]; + + ctx->blobToc = fopen("blobs.toc", PG_BINARY_R); + + _getBlobTocEntry(AH, &oid, fname); + + while(oid != 0) + { + StartRestoreBlob(AH, oid); + _PrintFileData(AH, fname, ropt); + EndRestoreBlob(AH, oid); + _getBlobTocEntry(AH, &oid, fname); + } + + fclose(ctx->blobToc); +} + + +static int _WriteByte(ArchiveHandle* AH, const int i) +{ + lclContext* ctx = (lclContext*)AH->formatData; + int res; + + res = fputc(i, AH->FH); + if (res != EOF) { + ctx->filePos += 1; + } + return res; +} + +static int _ReadByte(ArchiveHandle* AH) +{ + lclContext* ctx = (lclContext*)AH->formatData; + int res; + + res = fgetc(AH->FH); + if (res != EOF) { + ctx->filePos += 1; + } + return res; +} + +static int _WriteBuf(ArchiveHandle* AH, const void* buf, int len) +{ + lclContext* ctx = (lclContext*)AH->formatData; + int res; + res = fwrite(buf, 1, len, AH->FH); + ctx->filePos += res; + return res; +} + +static int _ReadBuf(ArchiveHandle* AH, void* buf, int len) +{ + lclContext* ctx = (lclContext*)AH->formatData; + int res; + + res = fread(buf, 1, len, AH->FH); + ctx->filePos += res; + return res; +} + +static void _CloseArchive(ArchiveHandle* AH) +{ + if (AH->mode == archModeWrite) { + WriteHead(AH); + WriteToc(AH); + fclose(AH->FH); + WriteDataChunks(AH); + } + + AH->FH = NULL; +} + + + +/* + * BLOB support + */ + +/* + * Called by the archiver when starting to save all BLOB DATA (not schema). + * This routine should save whatever format-specific information is needed + * to read the BLOBs back into memory. + * + * It is called just prior to the dumper's DataDumper routine. + * + * Optional, but strongly recommended. + * + */ +static void _StartBlobs(ArchiveHandle* AH, TocEntry* te) +{ + lclContext* ctx = (lclContext*)AH->formatData; + char fname[K_STD_BUF_SIZE]; + + sprintf(fname, "blobs.toc"); + ctx->blobToc = fopen(fname, PG_BINARY_W); + +} + +/* + * Called by the archiver when the dumper calls StartBlob. + * + * Mandatory. + * + * Must save the passed OID for retrieval at restore-time. + */ +static void _StartBlob(ArchiveHandle* AH, TocEntry* te, int oid) +{ + lclContext* ctx = (lclContext*)AH->formatData; + lclTocEntry* tctx = (lclTocEntry*)te->formatData; + char fmode[10]; + char fname[255]; + char *sfx; + + if (oid == 0) + die_horribly(AH, "%s: illegal OID for BLOB (%d)\n", progname, oid); + + if (AH->compression != 0) + sfx = ".gz"; + else + sfx = ""; + + sprintf(fmode, "wb%d", AH->compression); + sprintf(fname, "blob_%d.dat%s", oid, sfx); + + fprintf(ctx->blobToc, "%d %s\n", oid, fname); + +#ifdef HAVE_LIBZ + tctx->FH = gzopen(fname, fmode); +#else + tctx->FH = fopen(fname, PG_BINARY_W); +#endif + +} + +/* + * Called by the archiver when the dumper calls EndBlob. + * + * Optional. + * + */ +static void _EndBlob(ArchiveHandle* AH, TocEntry* te, int oid) +{ + lclTocEntry* tctx = (lclTocEntry*)te->formatData; + + GZCLOSE(tctx->FH); +} + +/* + * Called by the archiver when finishing saving all BLOB DATA. + * + * Optional. + * + */ +static void _EndBlobs(ArchiveHandle* AH, TocEntry* te) +{ + lclContext* ctx = (lclContext*)AH->formatData; + /* Write out a fake zero OID to mark end-of-blobs. */ + /* WriteInt(AH, 0); */ + + fclose(ctx->blobToc); + +} + + diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 8e8e81feb4..e4a47b9ef1 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -22,7 +22,7 @@ * * * IDENTIFICATION - * $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_dump.c,v 1.159 2000/07/17 03:05:20 tgl Exp $ + * $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_dump.c,v 1.160 2000/07/21 11:40:08 pjw Exp $ * * Modifications - 6/10/96 - dave@bensoft.com - version 1.13.dhb * @@ -61,12 +61,19 @@ * - Added a -Z option for compression level on compressed formats * - Restored '-f' in usage output * -*------------------------------------------------------------------------- + * + * Modifications - 17-Jul-2000 - Philip Warner pjw@rhyme.com.au + * - Support for BLOB output. + * - Sort archive by OID, put some items at end (out of OID order) + * + *------------------------------------------------------------------------- */ #include /* for getopt() */ #include +#include "pg_backup.h" + #include "postgres.h" #ifdef HAVE_GETOPT_H @@ -84,6 +91,7 @@ #include "catalog/pg_type.h" #include "libpq-fe.h" +#include #ifndef HAVE_STRDUP #include "strdup.h" #endif @@ -109,6 +117,9 @@ static void setMaxOid(Archive *fout); static void AddAcl(char *aclbuf, const char *keyword); static char *GetPrivileges(const char *s); +static int dumpBlobs(Archive *AH, char*, void*); + + extern char *optarg; extern int optind, opterr; @@ -268,14 +279,26 @@ dumpClasses_nodumpData(Archive *fout, char* oid, void *dctxv) if (oids == true) { - archprintf(fout, "COPY %s WITH OIDS FROM stdin;\n", - fmtId(classname, force_quotes)); + /* + * archprintf(fout, "COPY %s WITH OIDS FROM stdin;\n", + * fmtId(classname, force_quotes)); + * + * - Not used as of V1.3 (needs to be in ArchiveEntry call) + * + */ + sprintf(query, "COPY %s WITH OIDS TO stdout;\n", fmtId(classname, force_quotes)); } else { - archprintf(fout, "COPY %s FROM stdin;\n", fmtId(classname, force_quotes)); + /* + *archprintf(fout, "COPY %s FROM stdin;\n", fmtId(classname, force_quotes)); + * + * - Not used as of V1.3 (needs to be in ArchiveEntry call) + * + */ + sprintf(query, "COPY %s TO stdout;\n", fmtId(classname, force_quotes)); } res = PQexec(g_conn, query); @@ -452,19 +475,28 @@ dumpClasses_dumpData(Archive *fout, char* oid, void *dctxv) */ static void dumpClasses(const TableInfo *tblinfo, const int numTables, Archive *fout, - const char *onlytable, const bool oids) + const char *onlytable, const bool oids, const bool force_quotes) { int i; char *all_only; DataDumperPtr dumpFn; DumpContext *dumpCtx; + char *oidsPart; + char copyBuf[512]; + char *copyStmt; if (onlytable == NULL) all_only = "all"; else all_only = "only"; + if (oids == true) + oidsPart = "WITH OIDS "; + else + oidsPart = ""; + + if (g_verbose) fprintf(stderr, "%s dumping out the contents of %s %d table%s/sequence%s %s\n", g_comment_start, all_only, @@ -514,112 +546,28 @@ dumpClasses(const TableInfo *tblinfo, const int numTables, Archive *fout, dumpCtx->tblidx = i; dumpCtx->oids = oids; - if (!dumpData) + if (!dumpData) /* Dump/restore using COPY */ + { dumpFn = dumpClasses_nodumpData; /* dumpClasses_nodumpData(fout, classname, oids); */ - else + sprintf(copyBuf, "COPY %s %s FROM stdin;\n", fmtId(tblinfo[i].relname, force_quotes), + oidsPart); + copyStmt = copyBuf; + } + else /* Restore using INSERT */ + { dumpFn = dumpClasses_dumpData; /* dumpClasses_dumpData(fout, classname); */ + copyStmt = NULL; + } ArchiveEntry(fout, tblinfo[i].oid, fmtId(tblinfo[i].relname, false), - "TABLE DATA", NULL, "", "", tblinfo[i].usename, + "TABLE DATA", NULL, "", "", copyStmt, tblinfo[i].usename, dumpFn, dumpCtx); } } } -static void -prompt_for_password(char *username, char *password) -{ - char buf[512]; - int length; - -#ifdef HAVE_TERMIOS_H - struct termios t_orig, - t; -#endif - - fprintf(stderr, "Username: "); - fflush(stderr); - fgets(username, 100, stdin); - length = strlen(username); - /* skip rest of the line */ - if (length > 0 && username[length - 1] != '\n') - { - do - { - fgets(buf, 512, stdin); - } while (buf[strlen(buf) - 1] != '\n'); - } - if (length > 0 && username[length - 1] == '\n') - username[length - 1] = '\0'; - -#ifdef HAVE_TERMIOS_H - tcgetattr(0, &t); - t_orig = t; - t.c_lflag &= ~ECHO; - tcsetattr(0, TCSADRAIN, &t); -#endif - fprintf(stderr, "Password: "); - fflush(stderr); - fgets(password, 100, stdin); -#ifdef HAVE_TERMIOS_H - tcsetattr(0, TCSADRAIN, &t_orig); -#endif - - length = strlen(password); - /* skip rest of the line */ - if (length > 0 && password[length - 1] != '\n') - { - do - { - fgets(buf, 512, stdin); - } while (buf[strlen(buf) - 1] != '\n'); - } - if (length > 0 && password[length - 1] == '\n') - password[length - 1] = '\0'; - - fprintf(stderr, "\n\n"); -} - - -static void -check_database_version(bool ignoreVersion) -{ - PGresult *res; - double myversion; - const char *remoteversion_str; - double remoteversion; - - myversion = strtod(PG_VERSION, NULL); - res = PQexec(g_conn, "SELECT version()"); - if (!res || - PQresultStatus(res) != PGRES_TUPLES_OK || - PQntuples(res) != 1) - { - fprintf(stderr, "check_database_version(): command failed. Explanation from backend: '%s'.\n", PQerrorMessage(g_conn)); - exit_nicely(g_conn); - } - - remoteversion_str = PQgetvalue(res, 0, 0); - remoteversion = strtod(remoteversion_str + 11, NULL); - if (myversion != remoteversion) - { - fprintf(stderr, "Database version: %s\npg_dump version: %s\n", - remoteversion_str, PG_VERSION); - if (ignoreVersion) - fprintf(stderr, "Proceeding despite version mismatch.\n"); - else - { - fprintf(stderr, "Aborting because of version mismatch.\n" - "Use --ignore-version if you think it's safe to proceed anyway.\n"); - exit_nicely(g_conn); - } - } - PQclear(res); -} - - int main(int argc, char **argv) { @@ -634,20 +582,19 @@ main(int argc, char **argv) bool oids = false; TableInfo *tblinfo; int numTables; - char connect_string[512] = ""; - char tmp_string[128]; - char username[100]; - char password[100]; bool use_password = false; int compressLevel = -1; bool ignore_version = false; - int plainText = 0; - int outputClean = 0; + int plainText = 0; + int outputClean = 0; + int outputBlobs = 0; + RestoreOptions *ropt; #ifdef HAVE_GETOPT_LONG static struct option long_options[] = { {"data-only", no_argument, NULL, 'a'}, + {"blobs", no_argument, NULL, 'b' }, {"clean", no_argument, NULL, 'c'}, {"file", required_argument, NULL, 'f'}, {"format", required_argument, NULL, 'F'}, @@ -686,6 +633,12 @@ main(int argc, char **argv) else progname = strrchr(argv[0], SEP_CHAR) + 1; + /* Set defaulty options based on progname */ + if (strcmp(progname, "pg_backup") == 0) + { + format = "c"; + outputBlobs = 1; + } #ifdef HAVE_GETOPT_LONG while ((c = getopt_long(argc, argv, "acdDf:F:h:inNop:st:uvxzZ:V?", long_options, &optindex)) != -1) @@ -698,6 +651,10 @@ main(int argc, char **argv) case 'a': /* Dump data only */ dataOnly = true; break; + case 'b': /* Dump blobs */ + outputBlobs = true; + break; + case 'c': /* clean (i.e., drop) schema prior to * create */ outputClean = 1; @@ -843,7 +800,12 @@ main(int argc, char **argv) case 'p': case 'P': plainText = 1; - g_fout = CreateArchive(filename, archPlainText, 0); + g_fout = CreateArchive(filename, archNull, 0); + break; + + case 't': + case 'T': + g_fout = CreateArchive(filename, archTar, compressLevel); break; default: @@ -860,53 +822,13 @@ main(int argc, char **argv) exit(1); } - /* find database */ - if (!(dbname = argv[optind]) && - !(dbname = getenv("PGDATABASE"))) - { - fprintf(stderr, "%s: no database name specified\n", progname); - exit(1); - } + /* Let the archiver know how noisy to be */ + g_fout->verbose = g_verbose; - /* g_conn = PQsetdb(pghost, pgport, NULL, NULL, dbname); */ - if (pghost != NULL) - { - sprintf(tmp_string, "host=%s ", pghost); - strcat(connect_string, tmp_string); - } - if (pgport != NULL) - { - sprintf(tmp_string, "port=%s ", pgport); - strcat(connect_string, tmp_string); - } - if (dbname != NULL) - { - sprintf(tmp_string, "dbname=%s ", dbname); - strcat(connect_string, tmp_string); - } - if (use_password) - { - prompt_for_password(username, password); - strcat(connect_string, "authtype=password "); - sprintf(tmp_string, "user=%s ", username); - strcat(connect_string, tmp_string); - sprintf(tmp_string, "password=%s ", password); - strcat(connect_string, tmp_string); - MemSet(tmp_string, 0, sizeof(tmp_string)); - MemSet(password, 0, sizeof(password)); - } - g_conn = PQconnectdb(connect_string); - MemSet(connect_string, 0, sizeof(connect_string)); - /* check to see that the backend connection was successfully made */ - if (PQstatus(g_conn) == CONNECTION_BAD) - { - fprintf(stderr, "Connection to database '%s' failed.\n", dbname); - fprintf(stderr, "%s\n", PQerrorMessage(g_conn)); - exit_nicely(g_conn); - } + dbname = argv[optind]; - /* check for version mismatch */ - check_database_version(ignore_version); + /* Open the database using the Archiver, so it knows about it. Errors mean death */ + g_conn = ConnectDatabase(g_fout, dbname, pghost, pgport, use_password, ignore_version); /* * Start serializable transaction to dump consistent data @@ -916,17 +838,15 @@ main(int argc, char **argv) res = PQexec(g_conn, "begin"); if (!res || PQresultStatus(res) != PGRES_COMMAND_OK) - { - fprintf(stderr, "BEGIN command failed. Explanation from backend: '%s'.\n", PQerrorMessage(g_conn)); - exit_nicely(g_conn); - } + exit_horribly(g_fout, "BEGIN command failed. Explanation from backend: '%s'.\n", + PQerrorMessage(g_conn)); + PQclear(res); res = PQexec(g_conn, "set transaction isolation level serializable"); if (!res || PQresultStatus(res) != PGRES_COMMAND_OK) - { - fprintf(stderr, "SET TRANSACTION command failed. Explanation from backend: '%s'.\n", PQerrorMessage(g_conn)); - exit_nicely(g_conn); - } + exit_horribly(g_fout, "SET TRANSACTION command failed. Explanation from backend: '%s'.\n", + PQerrorMessage(g_conn)); + PQclear(res); } @@ -941,7 +861,12 @@ main(int argc, char **argv) tblinfo = dumpSchema(g_fout, &numTables, tablename, aclsSkip, oids, schemaOnly, dataOnly); if (!schemaOnly) - dumpClasses(tblinfo, numTables, g_fout, tablename, oids); + { + dumpClasses(tblinfo, numTables, g_fout, tablename, oids, force_quotes); + } + + if (outputBlobs) + ArchiveEntry(g_fout, "0", "BLOBS", "BLOBS", NULL, "", "", "", "", dumpBlobs, 0); if (!dataOnly) /* dump indexes and triggers at the end * for performance */ @@ -951,6 +876,15 @@ main(int argc, char **argv) dumpRules(g_fout, tablename, tblinfo, numTables); } + /* Now sort the output nicely */ + SortTocByOID(g_fout); + MoveToEnd(g_fout, "TABLE DATA"); + MoveToEnd(g_fout, "BLOBS"); + MoveToEnd(g_fout, "INDEX"); + MoveToEnd(g_fout, "TRIGGER"); + MoveToEnd(g_fout, "RULE"); + MoveToEnd(g_fout, "ACL"); + if (plainText) { ropt = NewRestoreOptions(); @@ -974,6 +908,92 @@ main(int argc, char **argv) } /* + * dumpBlobs: + * dump all blobs + * + */ + +#define loBufSize 16384 +#define loFetchSize 1000 + +static int +dumpBlobs(Archive *AH, char* junkOid, void *junkVal) +{ + PQExpBuffer oidQry = createPQExpBuffer(); + PQExpBuffer oidFetchQry = createPQExpBuffer(); + PGresult *res; + int i; + int loFd; + char buf[loBufSize]; + int cnt; + int blobOid; + + if (g_verbose) + fprintf(stderr, "%s saving BLOBs\n", g_comment_start); + + /* Cursor to get all BLOB tables */ + appendPQExpBuffer(oidQry, "Declare blobOid Cursor for SELECT oid from pg_class where relkind = 'l'"); + + res = PQexec(g_conn, oidQry->data); + if (!res || PQresultStatus(res) != PGRES_COMMAND_OK) + { + fprintf(stderr, "dumpBlobs(): Declare Cursor failed. Explanation from backend: '%s'.\n", PQerrorMessage(g_conn)); + exit_nicely(g_conn); + } + + /* Fetch for cursor */ + appendPQExpBuffer(oidFetchQry, "Fetch %d in blobOid", loFetchSize); + + do { + /* Do a fetch */ + PQclear(res); + res = PQexec(g_conn, oidFetchQry->data); + + if (!res || PQresultStatus(res) != PGRES_TUPLES_OK) + { + fprintf(stderr, "dumpBlobs(): Fetch Cursor failed. Explanation from backend: '%s'.\n", PQerrorMessage(g_conn)); + exit_nicely(g_conn); + } + + /* Process the tuples, if any */ + for (i = 0; i < PQntuples(res); i++) + { + blobOid = atoi(PQgetvalue(res, i, 0)); + /* Open the BLOB */ + loFd = lo_open(g_conn, blobOid, INV_READ); + if (loFd == -1) + { + fprintf(stderr, "dumpBlobs(): Could not open large object. " + "Explanation from backend: '%s'.\n", PQerrorMessage(g_conn)); + exit_nicely(g_conn); + } + + StartBlob(AH, blobOid); + + /* Now read it in chunks, sending data to archive */ + do { + cnt = lo_read(g_conn, loFd, buf, loBufSize); + if (cnt < 0) { + fprintf(stderr, "dumpBlobs(): Error reading large object. " + " Explanation from backend: '%s'.\n", PQerrorMessage(g_conn)); + exit_nicely(g_conn); + } + + WriteData(AH, buf, cnt); + + } while (cnt > 0); + + lo_close(g_conn, loFd); + + EndBlob(AH, blobOid); + + } + } while (PQntuples(res) > 0); + + return 1; +} + +/* * getTypes: * read all base types in the system catalogs and return them in the * TypeInfo* structure @@ -2409,7 +2429,7 @@ dumpComment(Archive *fout, const char *target, const char *oid) target, checkForQuote(PQgetvalue(res, 0, i_description))); ArchiveEntry(fout, oid, target, "COMMENT", NULL, query->data, "" /*Del*/, - "" /*Owner*/, NULL, NULL); + "" /* Copy */, "" /*Owner*/, NULL, NULL); } @@ -2542,7 +2562,7 @@ dumpTypes(Archive *fout, FuncInfo *finfo, int numFuncs, appendPQExpBuffer(q, ");\n"); ArchiveEntry(fout, tinfo[i].oid, fmtId(tinfo[i].typname, force_quotes), "TYPE", NULL, - q->data, delq->data, tinfo[i].usename, NULL, NULL); + q->data, delq->data, "", tinfo[i].usename, NULL, NULL); /*** Dump Type Comments ***/ @@ -2629,7 +2649,7 @@ dumpProcLangs(Archive *fout, FuncInfo *finfo, int numFuncs, lancompiler); ArchiveEntry(fout, PQgetvalue(res, i, i_oid), lanname, "PROCEDURAL LANGUAGE", - NULL, defqry->data, delqry->data, "", NULL, NULL); + NULL, defqry->data, delqry->data, "", "", NULL, NULL); free(lanname); free(lancompiler); @@ -2669,8 +2689,8 @@ dumpOneFunc(Archive *fout, FuncInfo *finfo, int i, PQExpBuffer fn = createPQExpBuffer(); PQExpBuffer delqry = createPQExpBuffer(); PQExpBuffer fnlist = createPQExpBuffer(); - PQExpBuffer asPart = createPQExpBuffer(); int j; + PQExpBuffer asPart = createPQExpBuffer(); char func_lang[NAMEDATALEN + 1]; PGresult *res; int nlangs; @@ -2703,7 +2723,7 @@ dumpOneFunc(Archive *fout, FuncInfo *finfo, int i, i_lanname = PQfnumber(res, "lanname"); - /* + /* * See backend/commands/define.c for details of how the 'AS' clause * is used. */ @@ -2751,7 +2771,7 @@ dumpOneFunc(Archive *fout, FuncInfo *finfo, int i, asPart->data, func_lang); ArchiveEntry(fout, finfo[i].oid, fn->data, "FUNCTION", NULL, q->data, delqry->data, - finfo[i].usename, NULL, NULL); + "", finfo[i].usename, NULL, NULL); /*** Dump Function Comments ***/ @@ -2870,7 +2890,7 @@ dumpOprs(Archive *fout, OprInfo *oprinfo, int numOperators, sort2->data); ArchiveEntry(fout, oprinfo[i].oid, oprinfo[i].oprname, "OPERATOR", NULL, - q->data, delq->data, oprinfo[i].usename, NULL, NULL); + q->data, delq->data, "", oprinfo[i].usename, NULL, NULL); } } @@ -2927,7 +2947,7 @@ dumpAggs(Archive *fout, AggInfo *agginfo, int numAggs, details->data); ArchiveEntry(fout, agginfo[i].oid, aggSig->data, "AGGREGATE", NULL, - q->data, delq->data, agginfo[i].usename, NULL, NULL); + q->data, delq->data, "", agginfo[i].usename, NULL, NULL); /*** Dump Aggregate Comments ***/ @@ -3096,7 +3116,7 @@ dumpACL(Archive *fout, TableInfo tbinfo) free(aclbuf); - ArchiveEntry(fout, tbinfo.oid, tbinfo.relname, "ACL", NULL, sql, "", "", NULL, NULL); + ArchiveEntry(fout, tbinfo.oid, tbinfo.relname, "ACL", NULL, sql, "", "", "", NULL, NULL); } @@ -3274,7 +3294,7 @@ dumpTables(Archive *fout, TableInfo *tblinfo, int numTables, if (!dataOnly) { ArchiveEntry(fout, tblinfo[i].oid, fmtId(tblinfo[i].relname, false), - "TABLE", NULL, q->data, delq->data, tblinfo[i].usename, + "TABLE", NULL, q->data, delq->data, "", tblinfo[i].usename, NULL, NULL); } @@ -3468,7 +3488,7 @@ dumpIndices(Archive *fout, IndInfo *indinfo, int numIndices, /* Dump Index Comments */ ArchiveEntry(fout, tblinfo[tableInd].oid, id1->data, "INDEX", NULL, q->data, delq->data, - tblinfo[tableInd].usename, NULL, NULL); + "", tblinfo[tableInd].usename, NULL, NULL); resetPQExpBuffer(q); appendPQExpBuffer(q, "INDEX %s", id1->data); @@ -3599,7 +3619,7 @@ setMaxOid(Archive *fout) pos = pos + snprintf(sql+pos, 1024-pos, "\\.\n"); pos = pos + snprintf(sql+pos, 1024-pos, "DROP TABLE pg_dump_oid;\n"); - ArchiveEntry(fout, "0", "Max OID", "", NULL, sql, "","", NULL, NULL); + ArchiveEntry(fout, "0", "Max OID", "", NULL, sql, "", "", "", NULL, NULL); } /* @@ -3750,7 +3770,7 @@ dumpSequence(Archive *fout, TableInfo tbinfo) } ArchiveEntry(fout, tbinfo.oid, fmtId(tbinfo.relname, force_quotes), "SEQUENCE", NULL, - query->data, delqry->data, tbinfo.usename, NULL, NULL); + query->data, delqry->data, "", tbinfo.usename, NULL, NULL); /* Dump Sequence Comments */ @@ -3779,7 +3799,7 @@ dumpTriggers(Archive *fout, const char *tablename, for (j = 0; j < tblinfo[i].ntrig; j++) { ArchiveEntry(fout, tblinfo[i].triggers[j].oid, tblinfo[i].triggers[j].tgname, - "TRIGGER", NULL, tblinfo[i].triggers[j].tgsrc, "", + "TRIGGER", NULL, tblinfo[i].triggers[j].tgsrc, "", "", tblinfo[i].usename, NULL, NULL); dumpComment(fout, tblinfo[i].triggers[j].tgcomment, tblinfo[i].triggers[j].oid); } @@ -3846,7 +3866,7 @@ dumpRules(Archive *fout, const char *tablename, ArchiveEntry(fout, PQgetvalue(res, i, i_oid), PQgetvalue(res, i, i_rulename), "RULE", NULL, PQgetvalue(res, i, i_definition), - "", "", NULL, NULL); + "", "", "", NULL, NULL); /* Dump rule comments */ diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c index 7b1a17c3f7..b39d8f7802 100644 --- a/src/bin/pg_dump/pg_restore.c +++ b/src/bin/pg_dump/pg_restore.c @@ -1,325 +1,371 @@ -/*------------------------------------------------------------------------- - * - * pg_restore.c - * pg_restore is an utility extracting postgres database definitions - * from a backup archive created by pg_dump using the archiver - * interface. - * - * pg_restore will read the backup archive and - * dump out a script that reproduces - * the schema of the database in terms of - * user-defined types - * user-defined functions - * tables - * indices - * aggregates - * operators - * ACL - grant/revoke - * - * the output script is SQL that is understood by PostgreSQL - * - * Basic process in a restore operation is: - * - * Open the Archive and read the TOC. - * Set flags in TOC entries, and *maybe* reorder them. - * Generate script to stdout - * Exit - * - * Copyright (c) 2000, Philip Warner - * Rights are granted to use this software in any way so long - * as this notice is not removed. - * - * The author is not responsible for loss or damages that may - * result from it's use. - * - * - * IDENTIFICATION - * - * Modifications - 28-Jun-2000 - pjw@rhyme.com.au - * - * Initial version. Command processing taken from original pg_dump. - * - *------------------------------------------------------------------------- - */ - -#include -#include -#include -#include - - -/* -#include "postgres.h" -#include "access/htup.h" -#include "catalog/pg_type.h" -#include "catalog/pg_language.h" -#include "catalog/pg_index.h" -#include "catalog/pg_trigger.h" -#include "libpq-fe.h" -*/ - -#include "pg_backup.h" - -#ifndef HAVE_STRDUP -#include "strdup.h" -#endif - -#ifdef HAVE_TERMIOS_H -#include -#endif - -#ifdef HAVE_GETOPT_H -#include -#else -#include -#endif - -/* Forward decls */ -static void usage(const char *progname); -static char* _cleanupName(char* name); - -typedef struct option optType; - -#ifdef HAVE_GETOPT_H -struct option cmdopts[] = { - { "clean", 0, NULL, 'c' }, - { "data-only", 0, NULL, 'a' }, - { "file", 1, NULL, 'f' }, - { "format", 1, NULL, 'F' }, - { "function", 2, NULL, 'p' }, - { "index", 2, NULL, 'i'}, - { "list", 0, NULL, 'l'}, - { "no-acl", 0, NULL, 'x' }, - { "oid-order", 0, NULL, 'o'}, - { "orig-order", 0, NULL, 'O' }, - { "rearrange", 0, NULL, 'r'}, - { "schema-only", 0, NULL, 's' }, - { "table", 2, NULL, 't'}, - { "trigger", 2, NULL, 'T' }, - { "use-list", 1, NULL, 'u'}, - { "verbose", 0, NULL, 'v' }, - { NULL, 0, NULL, 0} - }; -#endif - -int main(int argc, char **argv) -{ - RestoreOptions *opts; - char *progname; - int c; - Archive* AH; - char *fileSpec = NULL; - - opts = NewRestoreOptions(); - - progname = *argv; - -#ifdef HAVE_GETOPT_LONG - while ((c = getopt_long(argc, argv, "acf:F:i:loOp:st:T:u:vx", cmdopts, NULL)) != EOF) -#else - while ((c = getopt(argc, argv, "acf:F:i:loOp:st:T:u:vx")) != -1) -#endif - { - switch (c) - { - case 'a': /* Dump data only */ - opts->dataOnly = 1; - break; - case 'c': /* clean (i.e., drop) schema prior to - * create */ - opts->dropSchema = 1; - break; - case 'f': /* output file name */ - opts->filename = strdup(optarg); - break; - case 'F': - if (strlen(optarg) != 0) - opts->formatName = strdup(optarg); - break; - case 'o': - opts->oidOrder = 1; - break; - case 'O': - opts->origOrder = 1; - break; - case 'r': - opts->rearrange = 1; - break; - - case 'p': /* Function */ - opts->selTypes = 1; - opts->selFunction = 1; - opts->functionNames = _cleanupName(optarg); - break; - case 'i': /* Index */ - opts->selTypes = 1; - opts->selIndex = 1; - opts->indexNames = _cleanupName(optarg); - break; - case 'T': /* Trigger */ - opts->selTypes = 1; - opts->selTrigger = 1; - opts->triggerNames = _cleanupName(optarg); - break; - case 's': /* dump schema only */ - opts->schemaOnly = 1; - break; - case 't': /* Dump data for this table only */ - opts->selTypes = 1; - opts->selTable = 1; - opts->tableNames = _cleanupName(optarg); - break; - case 'l': /* Dump the TOC summary */ - opts->tocSummary = 1; - break; - - case 'u': /* input TOC summary file name */ - opts->tocFile = strdup(optarg); - break; - - case 'v': /* verbose */ - opts->verbose = 1; - break; - case 'x': /* skip ACL dump */ - opts->aclsSkip = 1; - break; - default: - usage(progname); - break; - } - } - - if (optind < argc) { - fileSpec = argv[optind]; - } else { - fileSpec = NULL; - } - - if (opts->formatName) { - - switch (opts->formatName[0]) { - - case 'c': - case 'C': - opts->format = archCustom; - break; - - case 'f': - case 'F': - opts->format = archFiles; - break; - - default: - fprintf(stderr, "%s: Unknown archive format '%s', please specify 'f' or 'c'\n", progname, opts->formatName); - exit (1); - } - } - - AH = OpenArchive(fileSpec, opts->format); - - if (opts->tocFile) - SortTocFromFile(AH, opts); - - if (opts->oidOrder) - SortTocByOID(AH); - else if (opts->origOrder) - SortTocByID(AH); - - if (opts->rearrange) { - MoveToEnd(AH, "TABLE DATA"); - MoveToEnd(AH, "INDEX"); - MoveToEnd(AH, "TRIGGER"); - MoveToEnd(AH, "RULE"); - MoveToEnd(AH, "ACL"); - } - - if (opts->tocSummary) { - PrintTOCSummary(AH, opts); - } else { - RestoreArchive(AH, opts); - } - - CloseArchive(AH); - - return 1; -} - -static void usage(const char *progname) -{ -#ifdef HAVE_GETOPT_LONG - fprintf(stderr, - "usage: %s [options] [backup file]\n" - " -a, --data-only \t dump out only the data, no schema\n" - " -c, --clean \t clean(drop) schema prior to create\n" - " -f filename \t script output filename\n" - " -F, --format {c|f} \t specify backup file format\n" - " -p, --function[=name] \t dump functions or named function\n" - " -i, --index[=name] \t dump indexes or named index\n" - " -l, --list \t dump summarized TOC for this file\n" - " -o, --oid-order \t dump in oid order\n" - " -O, --orig-order \t dump in original dump order\n" - " -r, --rearrange \t rearrange output to put indexes etc at end\n" - " -s, --schema-only \t dump out only the schema, no data\n" - " -t [table], --table[=table] \t dump for this table only\n" - " -T, --trigger[=name] \t dump triggers or named trigger\n" - " -u, --use-list filename \t use specified TOC for ordering output from this file\n" - " -v \t verbose\n" - " -x, --no-acl \t skip dumping of ACLs (grant/revoke)\n" - , progname); -#else - fprintf(stderr, - "usage: %s [options] [backup file]\n" - " -a \t dump out only the data, no schema\n" - " -c \t clean(drop) schema prior to create\n" - " -f filename NOT IMPLEMENTED \t script output filename\n" - " -F {c|f} \t specify backup file format\n" - " -p name \t dump functions or named function\n" - " -i name \t dump indexes or named index\n" - " -l \t dump summarized TOC for this file\n" - " -o \t dump in oid order\n" - " -O \t dump in original dump order\n" - " -r \t rearrange output to put indexes etc at end\n" - " -s \t dump out only the schema, no data\n" - " -t name \t dump for this table only\n" - " -T name \t dump triggers or named trigger\n" - " -u filename \t use specified TOC for ordering output from this file\n" - " -v \t verbose\n" - " -x \t skip dumping of ACLs (grant/revoke)\n" - , progname); -#endif - fprintf(stderr, - "\nIf [backup file] is not supplied, then standard input " - "is used.\n"); - fprintf(stderr, "\n"); - - exit(1); -} - -static char* _cleanupName(char* name) -{ - int i; - - if (!name) - return NULL; - - if (strlen(name) == 0) - return NULL; - - name = strdup(name); - - if (name[0] == '"') - { - strcpy(name, &name[1]); - if (*(name + strlen(name) - 1) == '"') - *(name + strlen(name) - 1) = '\0'; - } - /* otherwise, convert table name to lowercase... */ - else - { - for (i = 0; name[i]; i++) - if (isascii((unsigned char) name[i]) && isupper(name[i])) - name[i] = tolower(name[i]); - } - return name; -} - +/*------------------------------------------------------------------------- + * + * pg_restore.c + * pg_restore is an utility extracting postgres database definitions + * from a backup archive created by pg_dump using the archiver + * interface. + * + * pg_restore will read the backup archive and + * dump out a script that reproduces + * the schema of the database in terms of + * user-defined types + * user-defined functions + * tables + * indices + * aggregates + * operators + * ACL - grant/revoke + * + * the output script is SQL that is understood by PostgreSQL + * + * Basic process in a restore operation is: + * + * Open the Archive and read the TOC. + * Set flags in TOC entries, and *maybe* reorder them. + * Generate script to stdout + * Exit + * + * Copyright (c) 2000, Philip Warner + * Rights are granted to use this software in any way so long + * as this notice is not removed. + * + * The author is not responsible for loss or damages that may + * result from it's use. + * + * + * IDENTIFICATION + * + * Modifications - 28-Jun-2000 - pjw@rhyme.com.au + * + * Initial version. Command processing taken from original pg_dump. + * + *------------------------------------------------------------------------- + */ + +#include +#include +#include +#include + + +/* +#include "postgres.h" +#include "access/htup.h" +#include "catalog/pg_type.h" +#include "catalog/pg_language.h" +#include "catalog/pg_index.h" +#include "catalog/pg_trigger.h" +#include "libpq-fe.h" +*/ + +#include "pg_backup.h" + +#ifndef HAVE_STRDUP +#include "strdup.h" +#endif + +#ifdef HAVE_TERMIOS_H +#include +#endif + +#ifdef HAVE_GETOPT_H +#include +#else +#include +#endif + +/* Forward decls */ +static void usage(const char *progname); +static char* _cleanupName(char* name); + +typedef struct option optType; + +#ifdef HAVE_GETOPT_H +struct option cmdopts[] = { + { "clean", 0, NULL, 'c' }, + { "data-only", 0, NULL, 'a' }, + { "dbname", 1, NULL, 'd' }, + { "file", 1, NULL, 'f' }, + { "format", 1, NULL, 'F' }, + { "function", 2, NULL, 'P' }, + { "host", 1, NULL, 'h' }, + { "ignore-version", 0, NULL, 'i'}, + { "index", 2, NULL, 'I'}, + { "list", 0, NULL, 'l'}, + { "no-acl", 0, NULL, 'x' }, + { "port", 1, NULL, 'p' }, + { "oid-order", 0, NULL, 'o'}, + { "orig-order", 0, NULL, 'O' }, + { "password", 0, NULL, 'u' }, + { "rearrange", 0, NULL, 'r'}, + { "schema-only", 0, NULL, 's' }, + { "table", 2, NULL, 't'}, + { "trigger", 2, NULL, 'T' }, + { "use-list", 1, NULL, 'U'}, + { "verbose", 0, NULL, 'v' }, + { NULL, 0, NULL, 0} + }; +#endif + +int main(int argc, char **argv) +{ + RestoreOptions *opts; + char *progname; + int c; + Archive* AH; + char *fileSpec = NULL; + + opts = NewRestoreOptions(); + + progname = *argv; + +#ifdef HAVE_GETOPT_LONG + while ((c = getopt_long(argc, argv, "acd:f:F:h:i:loOp:st:T:u:U:vx", cmdopts, NULL)) != EOF) +#else + while ((c = getopt(argc, argv, "acd:f:F:h:i:loOp:st:T:u:U:vx")) != -1) +#endif + { + switch (c) + { + case 'a': /* Dump data only */ + opts->dataOnly = 1; + break; + case 'c': /* clean (i.e., drop) schema prior to + * create */ + opts->dropSchema = 1; + break; + case 'd': + if (strlen(optarg) != 0) + { + opts->dbname = strdup(optarg); + opts->useDB = 1; + } + break; + case 'f': /* output file name */ + opts->filename = strdup(optarg); + break; + case 'F': + if (strlen(optarg) != 0) + opts->formatName = strdup(optarg); + break; + case 'h': + if (strlen(optarg) != 0) + opts->pghost = strdup(optarg); + break; + case 'i': + opts->ignoreVersion = 1; + break; + case 'o': + opts->oidOrder = 1; + break; + case 'O': + opts->origOrder = 1; + break; + case 'p': + if (strlen(optarg) != 0) + opts->pgport = strdup(optarg); + break; + case 'r': + opts->rearrange = 1; + break; + case 'P': /* Function */ + opts->selTypes = 1; + opts->selFunction = 1; + opts->functionNames = _cleanupName(optarg); + break; + case 'I': /* Index */ + opts->selTypes = 1; + opts->selIndex = 1; + opts->indexNames = _cleanupName(optarg); + break; + case 'T': /* Trigger */ + opts->selTypes = 1; + opts->selTrigger = 1; + opts->triggerNames = _cleanupName(optarg); + break; + case 's': /* dump schema only */ + opts->schemaOnly = 1; + break; + case 't': /* Dump data for this table only */ + opts->selTypes = 1; + opts->selTable = 1; + opts->tableNames = _cleanupName(optarg); + break; + case 'l': /* Dump the TOC summary */ + opts->tocSummary = 1; + break; + + case 'u': + opts->requirePassword = 1; + break; + + case 'U': /* input TOC summary file name */ + opts->tocFile = strdup(optarg); + break; + + case 'v': /* verbose */ + opts->verbose = 1; + break; + case 'x': /* skip ACL dump */ + opts->aclsSkip = 1; + break; + default: + usage(progname); + break; + } + } + + if (optind < argc) { + fileSpec = argv[optind]; + } else { + fileSpec = NULL; + } + + if (opts->formatName) { + + switch (opts->formatName[0]) { + + case 'c': + case 'C': + opts->format = archCustom; + break; + + case 'f': + case 'F': + opts->format = archFiles; + break; + + case 't': + case 'T': + opts->format = archTar; + break; + + default: + fprintf(stderr, "%s: Unknown archive format '%s', please specify 't' or 'c'\n", + progname, opts->formatName); + exit (1); + } + } + + AH = OpenArchive(fileSpec, opts->format); + + /* Let the archiver know how noisy to be */ + AH->verbose = opts->verbose; + + if (opts->tocFile) + SortTocFromFile(AH, opts); + + if (opts->oidOrder) + SortTocByOID(AH); + else if (opts->origOrder) + SortTocByID(AH); + + if (opts->rearrange) { + MoveToStart(AH, ""); + MoveToEnd(AH, "TABLE DATA"); + MoveToEnd(AH, "BLOBS"); + MoveToEnd(AH, "INDEX"); + MoveToEnd(AH, "TRIGGER"); + MoveToEnd(AH, "RULE"); + MoveToEnd(AH, "ACL"); + } + + if (opts->tocSummary) { + PrintTOCSummary(AH, opts); + } else { + RestoreArchive(AH, opts); + } + + CloseArchive(AH); + + return 1; +} + +static void usage(const char *progname) +{ +#ifdef HAVE_GETOPT_LONG + fprintf(stderr, + "usage: %s [options] [backup file]\n" + " -a, --data-only \t dump out only the data, no schema\n" + " -d, --dbname \t specify database name\n" + " -c, --clean \t clean(drop) schema prior to create\n" + " -f filename \t script output filename\n" + " -F, --format {c|f} \t specify backup file format\n" + " -h, --host \t server host name\n" + " -i, --index[=name] \t dump indexes or named index\n" + " -l, --list \t dump summarized TOC for this file\n" + " -o, --oid-order \t dump in oid order\n" + " -O, --orig-order \t dump in original dump order\n" + " -p, --port \t server port number\n" + " -P, --function[=name] \t dump functions or named function\n" + " -r, --rearrange \t rearrange output to put indexes etc at end\n" + " -s, --schema-only \t dump out only the schema, no data\n" + " -t [table], --table[=table] \t dump for this table only\n" + " -T, --trigger[=name] \t dump triggers or named trigger\n" + " -u, --password \t use password authentication\n" + " -U, --use-list filename \t use specified TOC for ordering output from this file\n" + " -v, --verbose \t verbose\n" + " -x, --no-acl \t skip dumping of ACLs (grant/revoke)\n" + , progname); + +#else + fprintf(stderr, + "usage: %s [options] [backup file]\n" + " -a \t dump out only the data, no schema\n" + " -d, \t specify database name\n" + " -c \t clean(drop) schema prior to create\n" + " -f filename NOT IMPLEMENTED \t script output filename\n" + " -F {c|f} \t specify backup file format\n" + " -h, \t server host name\n" + " -i name \t dump indexes or named index\n" + " -l \t dump summarized TOC for this file\n" + " -o \t dump in oid order\n" + " -O \t dump in original dump order\n" + " -p \t server port number\n" + " -P name \t dump functions or named function\n" + " -r \t rearrange output to put indexes etc at end\n" + " -s \t dump out only the schema, no data\n" + " -t name \t dump for this table only\n" + " -T name \t dump triggers or named trigger\n" + " -u \t use password authentication\n" + " -U filename \t use specified TOC for ordering output from this file\n" + " -v \t verbose\n" + " -x \t skip dumping of ACLs (grant/revoke)\n" + , progname); +#endif + fprintf(stderr, + "\nIf [backup file] is not supplied, then standard input " + "is used.\n"); + fprintf(stderr, "\n"); + + exit(1); +} + +static char* _cleanupName(char* name) +{ + int i; + + if (!name) + return NULL; + + if (strlen(name) == 0) + return NULL; + + name = strdup(name); + + if (name[0] == '"') + { + strcpy(name, &name[1]); + if (*(name + strlen(name) - 1) == '"') + *(name + strlen(name) - 1) = '\0'; + } + /* otherwise, convert table name to lowercase... */ + else + { + for (i = 0; name[i]; i++) + if (isascii((unsigned char) name[i]) && isupper(name[i])) + name[i] = tolower(name[i]); + } + return name; +} + -- 2.11.0