Fossil

Check-in [d2d1a86f]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Improvements to error detection and reporting in the artifact parser. Add the test-parse-all-blobs command for verifying the artifact parser against historical repositories.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | forum-v2
Files: files | file ages | folders
SHA3-256: d2d1a86fa29f12a69735c90478d3638d4d50383d7134069dd5c7fe21c717cacb
User & Date: drh 2018-07-30 14:14:33.876
Context
2018-07-30
16:01
Add new security-audit checks for forum and "Announce" privileges. ... (check-in: 5d6fc967 user: drh tags: forum-v2)
14:14
Improvements to error detection and reporting in the artifact parser. Add the test-parse-all-blobs command for verifying the artifact parser against historical repositories. ... (check-in: d2d1a86f user: drh tags: forum-v2)
13:34
Merge backoffice enhancements from trunk. ... (check-in: bb50f0dc user: drh tags: forum-v2)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/manifest.c.
419
420
421
422
423
424
425
426


427
428
429
430
431
432
433
  ManifestText x;
  char cPrevType = 0;
  char cType;
  char *z;
  int n;
  char *zUuid;
  int sz = 0;
  int isRepeat, hasSelfRefTag = 0;


  static Bag seen;
  const char *zErr = 0;
  unsigned int m;
  unsigned int seenCard = 0;   /* Which card types have been seen */
  char zErrBuf[100];           /* Write error messages here */

  if( rid==0 ){







|
>
>







419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
  ManifestText x;
  char cPrevType = 0;
  char cType;
  char *z;
  int n;
  char *zUuid;
  int sz = 0;
  int isRepeat;
  int nSelfTag = 0;     /* Number of T cards referring to this manifest */
  int nSimpleTag = 0;   /* Number of T cards with "+" prefix */
  static Bag seen;
  const char *zErr = 0;
  unsigned int m;
  unsigned int seenCard = 0;   /* Which card types have been seen */
  char zErrBuf[100];           /* Write error messages here */

  if( rid==0 ){
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900

901
902
903
904
905
906
907
        if( zName==0 ) SYNTAX("missing name on T-card");
        zUuid = next_token(&x, &sz);
        if( zUuid==0 ) SYNTAX("missing artifact hash on T-card");
        zValue = next_token(&x, 0);
        if( zValue ) defossilize(zValue);
        if( hname_validate(zUuid, sz) ){
          /* A valid artifact hash */
          if( p->zEventId ) SYNTAX("non-self-referential T-card in technote");
        }else if( sz==1 && zUuid[0]=='*' ){
          zUuid = 0;
          hasSelfRefTag = 1;
          if( p->zEventId && zName[0]!='+' ){
            SYNTAX("propagating T-card in event");
          }
        }else{
          SYNTAX("malformed artifact hash on T-card");
        }
        defossilize(zName);
        if( zName[0]!='-' && zName[0]!='+' && zName[0]!='*' ){
          SYNTAX("T-card name does not begin with '-', '+', or '*'");
        }

        if( validate16(&zName[1], strlen(&zName[1])) ){
          /* Do not allow tags whose names look like a hash */
          SYNTAX("T-card name looks like a hexadecimal hash");
        }
        if( p->nTag>=p->nTagAlloc ){
          p->nTagAlloc = p->nTagAlloc*2 + 10;
          p->aTag = fossil_realloc(p->aTag, p->nTagAlloc*sizeof(p->aTag[0]) );







<


|
<
<
<







>







882
883
884
885
886
887
888

889
890
891



892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
        if( zName==0 ) SYNTAX("missing name on T-card");
        zUuid = next_token(&x, &sz);
        if( zUuid==0 ) SYNTAX("missing artifact hash on T-card");
        zValue = next_token(&x, 0);
        if( zValue ) defossilize(zValue);
        if( hname_validate(zUuid, sz) ){
          /* A valid artifact hash */

        }else if( sz==1 && zUuid[0]=='*' ){
          zUuid = 0;
          nSelfTag++;



        }else{
          SYNTAX("malformed artifact hash on T-card");
        }
        defossilize(zName);
        if( zName[0]!='-' && zName[0]!='+' && zName[0]!='*' ){
          SYNTAX("T-card name does not begin with '-', '+', or '*'");
        }
        if( zName[0]=='+' ) nSimpleTag++;
        if( validate16(&zName[1], strlen(&zName[1])) ){
          /* Do not allow tags whose names look like a hash */
          SYNTAX("T-card name looks like a hexadecimal hash");
        }
        if( p->nTag>=p->nTagAlloc ){
          p->nTagAlloc = p->nTagAlloc*2 + 10;
          p->aTag = fossil_realloc(p->aTag, p->nTagAlloc*sizeof(p->aTag[0]) );
1016
1017
1018
1019
1020
1021
1022













1023
1024
1025
1026
1027
1028
1029
                     azNameOfMType[p->type-1]);
    zErr = zErrBuf;
    goto manifest_syntax_error;
  }

  /* Additional checks based on artifact type */
  switch( p->type ){













    case CFTYPE_FORUM: {
      if( p->zThreadTitle && p->zInReplyTo ){
        SYNTAX("cannot have I-card and H-card in a forum post");
      }
      if( p->nParent>1 ) SYNTAX("too many arguments to P-card");
      break;
    }







>
>
>
>
>
>
>
>
>
>
>
>
>







1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
                     azNameOfMType[p->type-1]);
    zErr = zErrBuf;
    goto manifest_syntax_error;
  }

  /* Additional checks based on artifact type */
  switch( p->type ){
    case CFTYPE_CONTROL: {
      if( nSelfTag ) SYNTAX("self-referential T-card in control artifact");
      break;
    }
    case CFTYPE_EVENT: {
      if( p->nTag!=nSelfTag ){
        SYNTAX("non-self-referential T-card in technote");
      }
      if( p->nTag!=nSimpleTag ){
        SYNTAX("T-card with '*' or '-' in technote");
      }
      break;
    }
    case CFTYPE_FORUM: {
      if( p->zThreadTitle && p->zInReplyTo ){
        SYNTAX("cannot have I-card and H-card in a forum post");
      }
      if( p->nParent>1 ) SYNTAX("too many arguments to P-card");
      break;
    }
1097
1098
1099
1100
1101
1102
1103
1104

1105
1106
1107
1108
1109
1110
1111
}

/*
** COMMAND: test-parse-manifest
**
** Usage: %fossil test-parse-manifest FILENAME ?N?
**
** Parse the manifest and discarded.  Use for testing only.

*/
void manifest_test_parse_cmd(void){
  Manifest *p;
  Blob b;
  int i;
  int n = 1;
  sqlite3_open(":memory:", &g.db);







|
>







1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
}

/*
** COMMAND: test-parse-manifest
**
** Usage: %fossil test-parse-manifest FILENAME ?N?
**
** Parse the manifest(s) given on the command-line and report any
** errors.  If the N argument is given, run the parsing N times.
*/
void manifest_test_parse_cmd(void){
  Manifest *p;
  Blob b;
  int i;
  int n = 1;
  sqlite3_open(":memory:", &g.db);
1121
1122
1123
1124
1125
1126
1127





































1128
1129
1130
1131
1132
1133
1134
    blob_zero(&err);
    p = manifest_parse(&b2, 0, &err);
    if( p==0 ) fossil_print("ERROR: %s\n", blob_str(&err));
    blob_reset(&err);
    manifest_destroy(p);
  }
}






































/*
** Fetch the baseline associated with the delta-manifest p.
** Return 0 on success.  If unable to parse the baseline,
** throw an error.  If the baseline is a manifest, throw an
** error if throwError is true, or record that p is an orphan
** and return 1 if throwError is false.







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
    blob_zero(&err);
    p = manifest_parse(&b2, 0, &err);
    if( p==0 ) fossil_print("ERROR: %s\n", blob_str(&err));
    blob_reset(&err);
    manifest_destroy(p);
  }
}

/*
** COMMAND: test-parse-all-blobs
**
** Usage: %fossil test-parse-all-blobs
**
** Parse all entries in the BLOB table that are believed to be non-data
** artifacts and report any errors.  Run this test command on historical
** repositories after making any changes to the manifest_parse()
** implementation to confirm that the changes did not break anything.
*/
void manifest_test_parse_all_blobs_cmd(void){
  Manifest *p;
  Blob err;
  Stmt q;
  int nTest = 0;
  int nErr = 0;
  db_find_and_open_repository(0, 0);
  verify_all_options();
  db_prepare(&q, "SELECT DISTINCT objid FROM EVENT");
  while( db_step(&q)==SQLITE_ROW ){
    int id = db_column_int(&q,0);
    fossil_print("Checking %d       \r", id);
    nTest++;
    fflush(stdout);
    blob_init(&err, 0, 0);
    p = manifest_get(id, CFTYPE_ANY, &err);
    if( p==0 ){
      fossil_print("%d ERROR: %s\n", id, blob_str(&err));
      nErr++;
    }
    blob_reset(&err);
    manifest_destroy(p);
  }
  db_finalize(&q);
  fossil_print("%d tests with %d errors\n", nTest, nErr);
}

/*
** Fetch the baseline associated with the delta-manifest p.
** Return 0 on success.  If unable to parse the baseline,
** throw an error.  If the baseline is a manifest, throw an
** error if throwError is true, or record that p is an orphan
** and return 1 if throwError is false.
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
      p->rDate, rid, p->zUser, blob_str(&comment)+1
    );
    blob_reset(&comment);
  }
  if( p->type==CFTYPE_FORUM ){
    int froot, fprev, firt;
    char *zFType;
    const char *zTitle;
    schema_forum();
    froot = p->zThreadRoot ? uuid_to_rid(p->zThreadRoot, 1) : rid;
    fprev = p->nParent ? uuid_to_rid(p->azParent[0],1) : 0;
    firt = p->zInReplyTo ? uuid_to_rid(p->zInReplyTo,1) : 0;
    db_multi_exec(
      "INSERT INTO forumpost(fpid,froot,fprev,firt,fmtime)"
      "VALUES(%d,%d,nullif(%d,0),nullif(%d,0),%.17g)",







|







2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
      p->rDate, rid, p->zUser, blob_str(&comment)+1
    );
    blob_reset(&comment);
  }
  if( p->type==CFTYPE_FORUM ){
    int froot, fprev, firt;
    char *zFType;
    char *zTitle;
    schema_forum();
    froot = p->zThreadRoot ? uuid_to_rid(p->zThreadRoot, 1) : rid;
    fprev = p->nParent ? uuid_to_rid(p->azParent[0],1) : 0;
    firt = p->zInReplyTo ? uuid_to_rid(p->zInReplyTo,1) : 0;
    db_multi_exec(
      "INSERT INTO forumpost(fpid,froot,fprev,firt,fmtime)"
      "VALUES(%d,%d,nullif(%d,0),nullif(%d,0),%.17g)",