Corrupted memos

doc/pgdbf.man

@@ -4,7 +4,7 @@ pgdbf \- convert XBase / FoxPro tables to PostgreSQL
 
 .SH SYNOPSIS
 .B pgdbf
-[\-cCdDeEhqQtTuU] [-m memofile] filename [indexcolumn ...]
+[\-cCdDeEhqQtTuU] [-w excesslength] [-m memofile] filename [indexcolumn ...]
 
 .SH DESCRIPTION
 PgDBF is a program for converting XBase databases - particularly FoxPro
@@ -156,6 +156,14 @@ statement to clear the contents of a table before copying data into it.
 Suppress the
 .B TRUNCATE TABLE
 statement. Default.
+.TP
+.B -w excesslength
+Warn of corrupted MEMO fields, by reporting errors on the output and through
+standard error. excesslenght specifies a MEMO length beyond which a MEMO record
+will be considered corrupt. A value of 0 (zero) will disable this heuristic,
+but the other warnings will still be issued (invalid start offset, invalid type,
+zero length and length beyond end of file). Conversion will not stop when warnings
+are issued.
 
 .SH "OPTION NOTES"
 The

src/pgdbf.c

@@ -30,7 +30,7 @@
 #include <sys/types.h>
 
 #include "pgdbf.h"
-#define STANDARDOPTS "cCdDeEhm:i:nNpPqQrRtTuU"
+#define STANDARDOPTS "cCdDeEhm:i:nNpPqQrRtTuUw:"
 
 int main(int argc, char **argv) {
     /* Describing the DBF file */
@@ -66,6 +66,9 @@ int main(int argc, char **argv) {
     size_t       memofilesize;
     size_t       memorecordoffset;
 
+    int32_t      memolength;
+    int32_t      memotype;
+
     /* Processing and misc */
     IGNFIELD     *ignorefields;
     char *istr;
@@ -112,6 +115,8 @@ int main(int argc, char **argv) {
     int     optusetransaction = 1;
     int     optusetruncatetable = 0;
     int     opttrimpadding = 1;
+    int     optmemowarn = 0;
+    uint32_t optmemowarnexcess = 0;
 
     /* Describing the PostgreSQL table */
     char *tablename;
@@ -220,6 +225,10 @@ int main(int argc, char **argv) {
         case 'U':
             optusetruncatetable = 0;
             break;
+        case 'w':
+            optmemowarn = 1;
+            sscanf (optarg, "%u", &optmemowarnexcess);
+            break;
         case 'h':
         default:
             /* If we got here because someone requested '-h', exit
@@ -238,9 +247,9 @@ int main(int argc, char **argv) {
     if(optexitcode != -1) {
         printf(
 #if defined(HAVE_ICONV)
-               "Usage: %s [-cCdDeEhtTuU] [-s encoding] [-m memofilename] [-i fieldname1,fieldname2,fieldnameN] filename [indexcolumn ...]\n"
+               "Usage: %s [-cCdDeEhtTuU] [-w excesslength] [-s encoding] [-m memofilename] [-i fieldname1,fieldname2,fieldnameN] filename [indexcolumn ...]\n"
 #else
-               "Usage: %s [-cCdDeEhtTuU] [-m memofilename] [-i fieldname1,fieldname2,fieldnameN] filename [indexcolumn ...]\n"
+               "Usage: %s [-cCdDeEhtTuU] [-w excesslength] [-m memofilename] [-i fieldname1,fieldname2,fieldnameN] filename [indexcolumn ...]\n"
 #endif
                "Convert the named XBase file into PostgreSQL format\n"
                "\n"
@@ -268,6 +277,7 @@ int main(int argc, char **argv) {
                "  -T  do not use an enclosing transaction\n"
                "  -u  issue a 'TRUNCATE' command before inserting data\n"
                "  -U  do not issue a 'TRUNCATE' command before inserting data (default)\n"
+               "  -w  warn of corrupted MEMO fields, excesslenght 0 disables check for excess length\n"
                "\n"
 #if defined(HAVE_ICONV)
                "If you don't specify an encoding via '-s', the data will be printed as is.\n"
@@ -769,18 +779,53 @@ int main(int argc, char **argv) {
                             s++;
                         }
                     }
+
                     if(memoblocknumber) {
                         memorecordoffset = memoblocksize * memoblocknumber;
                         if(memorecordoffset >= memofilesize) {
-                            exitwitherror("A memo record past the end of the memofile was requested", 0);
+                          if(optmemowarn) {
+                              printf("CORRUPT:OFFSET::Memo record past end of memofile requested");
+                              fprintf(stderr, "CORRUPT:OFFSET::Memo record past end of memofile requested\n");
+                              break;
+                          }
+                          exitwitherror("A memo record past the end of the memofile was requested", 0);
                         }
+
                         memorecord = memomap + memorecordoffset;
                         if(memofileisdbase3) {
                             t = strchr(memorecord, 0x1A);
                             safeprintbuf(memorecord, t - memorecord, opttrimpadding);
                         } else {
-                            safeprintbuf(memorecord + 8, sbigint32_t(memorecord + 4), opttrimpadding);
+                            memotype = sbigint32_t(memorecord);
+                            if(optmemowarn && memotype != MEMOTYPETEXT && memotype != MEMOTYPEPICTURE) {
+                                printf("CORRUPT:TYPE:type 0x%08xd,offset %lu:Only types 0 (PICTURE) and 1 (TEXT) allowed", memotype, memorecordoffset);
+                                fprintf(stderr, "CORRUPT:TYPE:type 0x%08xd,offset %lu:Only types 0 (PICTURE) and 1 (TEXT) allowed\n", memotype, memorecordoffset);
+                                break;
+                            }
+                          exitwitherror("A memo record with an invalid type was specified", 0);
                         }
+
+                        memolength = sbigint32_t(memorecord + 4);
+                        if(optmemowarn && memolength == 0) {
+                             printf("CORRUPT:ZERO:offset %lu:Memo of length 0 specified", memorecordoffset);
+                             fprintf(stderr, "CORRUPT:ZERO:offset %lu:Memo of length 0 specified\n", memorecordoffset);
+                            break;
+                        }
+                        if(optmemowarn && optmemowarnexcess && memolength > optmemowarnexcess) {
+                             printf("CORRUPT:EXCESS:length %d,offset %lu:Memo of excess length specified", memolength, memorecordoffset);
+                             fprintf(stderr, "CORRUPT:EXCESS:length %d,offset %lu:Memo of excess length specified\n", memolength, memorecordoffset);
+                            break;
+                        }
+                        if(memorecordoffset + 8 + memolength > memofilesize) {
+                            if(optmemowarn) {
+                                 printf("CORRUPT:TOOLONG:length %d,offset %lu:Memo of length past the end of memofile specified", memolength, memorecordoffset);
+                                 fprintf(stderr, "CORRUPT:TOOLONG:length %d,offset %lu:Memo of length past the end of memofile  specified\n", memolength, memorecordoffset);
+                                break;
+                            }
+                            exitwitherror("A memo record with a length past the end of the memofile was specified", 0);
+                        }
+
+                        safeprintbuf(memorecord + 8, memolength, opttrimpadding);
                     }
                     break;
                 case 'F':

src/pgdbf.h

@@ -46,6 +46,9 @@
 #define NUMERICMEMOSTYLE 0
 #define PACKEDMEMOSTYLE 1
 
+#define MEMOTYPEPICTURE 0
+#define MEMOTYPETEXT 1
+
 /* Don't edit this! It's defined in the XBase specification. */
 #define XBASEFIELDNAMESIZE 11
 
@@ -244,6 +247,9 @@ static char* convertcharset(const char* inputstring, size_t* inputsize)
 #endif
 
 static void exitwitherror(const char *message, const int systemerror) {
+    /* Flush the standard output in case data is still buffered */
+    fflush(stdout);
+
     /* Print the given error message to stderr, then exit.  If systemerror
      * is true, then use perror to explain the value in errno. */
     if(systemerror) {