Fossil

Check-in [d2d1a86f]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Improvements to error detection and reporting in the artifact parser. Add the test-parse-all-blobs command for verifying the artifact parser against historical repositories.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | forum-v2
Files: files | file ages | folders
SHA3-256: d2d1a86fa29f12a69735c90478d3638d4d50383d7134069dd5c7fe21c717cacb
User & Date: drh 2018-07-30 14:14:33
Context
2018-07-30
16:01
Add new security-audit checks for forum and "Announce" privileges. check-in: 5d6fc967 user: drh tags: forum-v2
14:14
Improvements to error detection and reporting in the artifact parser. Add the test-parse-all-blobs command for verifying the artifact parser against historical repositories. check-in: d2d1a86f user: drh tags: forum-v2
13:34
Merge backoffice enhancements from trunk. check-in: bb50f0dc user: drh tags: forum-v2
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/manifest.c.

   419    419     ManifestText x;
   420    420     char cPrevType = 0;
   421    421     char cType;
   422    422     char *z;
   423    423     int n;
   424    424     char *zUuid;
   425    425     int sz = 0;
   426         -  int isRepeat, hasSelfRefTag = 0;
          426  +  int isRepeat;
          427  +  int nSelfTag = 0;     /* Number of T cards referring to this manifest */
          428  +  int nSimpleTag = 0;   /* Number of T cards with "+" prefix */
   427    429     static Bag seen;
   428    430     const char *zErr = 0;
   429    431     unsigned int m;
   430    432     unsigned int seenCard = 0;   /* Which card types have been seen */
   431    433     char zErrBuf[100];           /* Write error messages here */
   432    434   
   433    435     if( rid==0 ){
................................................................................
   880    882           if( zName==0 ) SYNTAX("missing name on T-card");
   881    883           zUuid = next_token(&x, &sz);
   882    884           if( zUuid==0 ) SYNTAX("missing artifact hash on T-card");
   883    885           zValue = next_token(&x, 0);
   884    886           if( zValue ) defossilize(zValue);
   885    887           if( hname_validate(zUuid, sz) ){
   886    888             /* A valid artifact hash */
   887         -          if( p->zEventId ) SYNTAX("non-self-referential T-card in technote");
   888    889           }else if( sz==1 && zUuid[0]=='*' ){
   889    890             zUuid = 0;
   890         -          hasSelfRefTag = 1;
   891         -          if( p->zEventId && zName[0]!='+' ){
   892         -            SYNTAX("propagating T-card in event");
   893         -          }
          891  +          nSelfTag++;
   894    892           }else{
   895    893             SYNTAX("malformed artifact hash on T-card");
   896    894           }
   897    895           defossilize(zName);
   898    896           if( zName[0]!='-' && zName[0]!='+' && zName[0]!='*' ){
   899    897             SYNTAX("T-card name does not begin with '-', '+', or '*'");
   900    898           }
          899  +        if( zName[0]=='+' ) nSimpleTag++;
   901    900           if( validate16(&zName[1], strlen(&zName[1])) ){
   902    901             /* Do not allow tags whose names look like a hash */
   903    902             SYNTAX("T-card name looks like a hexadecimal hash");
   904    903           }
   905    904           if( p->nTag>=p->nTagAlloc ){
   906    905             p->nTagAlloc = p->nTagAlloc*2 + 10;
   907    906             p->aTag = fossil_realloc(p->aTag, p->nTagAlloc*sizeof(p->aTag[0]) );
................................................................................
  1016   1015                        azNameOfMType[p->type-1]);
  1017   1016       zErr = zErrBuf;
  1018   1017       goto manifest_syntax_error;
  1019   1018     }
  1020   1019   
  1021   1020     /* Additional checks based on artifact type */
  1022   1021     switch( p->type ){
         1022  +    case CFTYPE_CONTROL: {
         1023  +      if( nSelfTag ) SYNTAX("self-referential T-card in control artifact");
         1024  +      break;
         1025  +    }
         1026  +    case CFTYPE_EVENT: {
         1027  +      if( p->nTag!=nSelfTag ){
         1028  +        SYNTAX("non-self-referential T-card in technote");
         1029  +      }
         1030  +      if( p->nTag!=nSimpleTag ){
         1031  +        SYNTAX("T-card with '*' or '-' in technote");
         1032  +      }
         1033  +      break;
         1034  +    }
  1023   1035       case CFTYPE_FORUM: {
  1024   1036         if( p->zThreadTitle && p->zInReplyTo ){
  1025   1037           SYNTAX("cannot have I-card and H-card in a forum post");
  1026   1038         }
  1027   1039         if( p->nParent>1 ) SYNTAX("too many arguments to P-card");
  1028   1040         break;
  1029   1041       }
................................................................................
  1097   1109   }
  1098   1110   
  1099   1111   /*
  1100   1112   ** COMMAND: test-parse-manifest
  1101   1113   **
  1102   1114   ** Usage: %fossil test-parse-manifest FILENAME ?N?
  1103   1115   **
  1104         -** Parse the manifest and discarded.  Use for testing only.
         1116  +** Parse the manifest(s) given on the command-line and report any
         1117  +** errors.  If the N argument is given, run the parsing N times.
  1105   1118   */
  1106   1119   void manifest_test_parse_cmd(void){
  1107   1120     Manifest *p;
  1108   1121     Blob b;
  1109   1122     int i;
  1110   1123     int n = 1;
  1111   1124     sqlite3_open(":memory:", &g.db);
................................................................................
  1121   1134       blob_zero(&err);
  1122   1135       p = manifest_parse(&b2, 0, &err);
  1123   1136       if( p==0 ) fossil_print("ERROR: %s\n", blob_str(&err));
  1124   1137       blob_reset(&err);
  1125   1138       manifest_destroy(p);
  1126   1139     }
  1127   1140   }
         1141  +
         1142  +/*
         1143  +** COMMAND: test-parse-all-blobs
         1144  +**
         1145  +** Usage: %fossil test-parse-all-blobs
         1146  +**
         1147  +** Parse all entries in the BLOB table that are believed to be non-data
         1148  +** artifacts and report any errors.  Run this test command on historical
         1149  +** repositories after making any changes to the manifest_parse()
         1150  +** implementation to confirm that the changes did not break anything.
         1151  +*/
         1152  +void manifest_test_parse_all_blobs_cmd(void){
         1153  +  Manifest *p;
         1154  +  Blob err;
         1155  +  Stmt q;
         1156  +  int nTest = 0;
         1157  +  int nErr = 0;
         1158  +  db_find_and_open_repository(0, 0);
         1159  +  verify_all_options();
         1160  +  db_prepare(&q, "SELECT DISTINCT objid FROM EVENT");
         1161  +  while( db_step(&q)==SQLITE_ROW ){
         1162  +    int id = db_column_int(&q,0);
         1163  +    fossil_print("Checking %d       \r", id);
         1164  +    nTest++;
         1165  +    fflush(stdout);
         1166  +    blob_init(&err, 0, 0);
         1167  +    p = manifest_get(id, CFTYPE_ANY, &err);
         1168  +    if( p==0 ){
         1169  +      fossil_print("%d ERROR: %s\n", id, blob_str(&err));
         1170  +      nErr++;
         1171  +    }
         1172  +    blob_reset(&err);
         1173  +    manifest_destroy(p);
         1174  +  }
         1175  +  db_finalize(&q);
         1176  +  fossil_print("%d tests with %d errors\n", nTest, nErr);
         1177  +}
  1128   1178   
  1129   1179   /*
  1130   1180   ** Fetch the baseline associated with the delta-manifest p.
  1131   1181   ** Return 0 on success.  If unable to parse the baseline,
  1132   1182   ** throw an error.  If the baseline is a manifest, throw an
  1133   1183   ** error if throwError is true, or record that p is an orphan
  1134   1184   ** and return 1 if throwError is false.
................................................................................
  2464   2514         p->rDate, rid, p->zUser, blob_str(&comment)+1
  2465   2515       );
  2466   2516       blob_reset(&comment);
  2467   2517     }
  2468   2518     if( p->type==CFTYPE_FORUM ){
  2469   2519       int froot, fprev, firt;
  2470   2520       char *zFType;
  2471         -    const char *zTitle;
         2521  +    char *zTitle;
  2472   2522       schema_forum();
  2473   2523       froot = p->zThreadRoot ? uuid_to_rid(p->zThreadRoot, 1) : rid;
  2474   2524       fprev = p->nParent ? uuid_to_rid(p->azParent[0],1) : 0;
  2475   2525       firt = p->zInReplyTo ? uuid_to_rid(p->zInReplyTo,1) : 0;
  2476   2526       db_multi_exec(
  2477   2527         "INSERT INTO forumpost(fpid,froot,fprev,firt,fmtime)"
  2478   2528         "VALUES(%d,%d,nullif(%d,0),nullif(%d,0),%.17g)",