Fossil

Check-in [043d63d4]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Additional speed improvements for clone and rebuild.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 043d63d4aa79d9558729452b3eba280e756f00fe
User & Date: drh 2008-03-08 19:42:53
Context
2008-03-10
16:46
Fix a bug introduced by recent optimization efforts. check-in: f763b841 user: drh tags: trunk
2008-03-08
19:42
Additional speed improvements for clone and rebuild. check-in: 043d63d4 user: drh tags: trunk
18:59
Speed enhancement in the findSrcid() routine of content.c. Allow 5 digit numbers on counts while syncing. check-in: 8010bb41 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/content.c.

362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
...
380
381
382
383
384
385
386

387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
...
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428

429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
...
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460

461
462
463
464
465
466
467
...
483
484
485
486
487
488
489
490
491
492

493
























494
495

496
497
498
499
500
501
502
/*
** Write content into the database.  Return the record ID.  If the
** content is already in the database, just return the record ID.
**
** If srcId is specified, then pBlob is delta content from
** the srcId record.  srcId might be a phantom.
**
** A phantom is written if pBlob==0.  If pBlob==0 or if srcId is
** specified then the UUID is set to zUuid.  Otherwise zUuid is
** ignored.  In the future this might change such that the content
** hash is checked against zUuid to make sure it is correct.
**
** If the record already exists but is a phantom, the pBlob content
** is inserted and the phatom becomes a real record.
*/
int content_put(Blob *pBlob, const char *zUuid, int srcId){
................................................................................
  Stmt s1;
  Blob cmpr;
  Blob hash;
  int markAsUnclustered = 0;
  int isDephantomize = 0;
  
  assert( g.repositoryOpen );

  if( pBlob && srcId==0 ){
    sha1sum_blob(pBlob, &hash);
  }else{
    blob_init(&hash, zUuid, -1);
  }
  if( pBlob==0 ){
    size = -1;
  }else{
    size = blob_size(pBlob);
  }
  db_begin_transaction();

  /* Check to see if the entry already exists and if it does whether
  ** or not the entry is a phantom
  */
  db_prepare(&s1, "SELECT rid, size FROM blob WHERE uuid=%B", &hash);
  if( db_step(&s1)==SQLITE_ROW ){
................................................................................
  }else{
    rid = 0;  /* No entry with the same UUID currently exists */
    markAsUnclustered = 1;
  }
  db_finalize(&s1);

  /* Construct a received-from ID if we do not already have one */
  if( g.rcvid==0 && pBlob!=0 ){
    db_multi_exec(
       "INSERT INTO rcvfrom(uid, mtime, nonce, ipaddr)"
       "VALUES(%d, julianday('now'), %Q, %Q)",
       g.userUid, g.zNonce, g.zIpAddr
    );
    g.rcvid = db_last_insert_rowid();
  }


  if( rid>0 ){
    /* We are just adding data to a phantom */
    assert( pBlob!=0 );
    db_prepare(&s1,
      "UPDATE blob SET rcvid=%d, size=%d, content=:data WHERE rid=%d",
       g.rcvid, size, rid
    );
    blob_compress(pBlob, &cmpr);
    db_bind_blob(&s1, ":data", &cmpr);
    db_exec(&s1);
    db_multi_exec("DELETE FROM phantom WHERE rid=%d", rid);
    if( srcId==0 || content_is_available(srcId) ){
      isDephantomize = 1;
      content_mark_available(rid);
    }
................................................................................
  }else{
    /* We are creating a new entry */
    db_prepare(&s1,
      "INSERT INTO blob(rcvid,size,uuid,content)"
      "VALUES(%d,%d,'%b',:data)",
       g.rcvid, size, &hash
    );
    if( pBlob ){
      blob_compress(pBlob, &cmpr);
      db_bind_blob(&s1, ":data", &cmpr);
    }
    db_exec(&s1);
    rid = db_last_insert_rowid();
    if( !pBlob ){
      db_multi_exec("INSERT OR IGNORE INTO phantom VALUES(%d)", rid);
    }
  }


  /* If the srcId is specified, then the data we just added is
  ** really a delta.  Record this fact in the delta table.
  */
  if( srcId ){
    db_multi_exec("REPLACE INTO delta(rid,srcid) VALUES(%d,%d)", rid, srcId);
  }
................................................................................
  /* Finish the transaction and cleanup */
  db_finalize(&s1);
  db_end_transaction(0);
  blob_reset(&hash);

  /* Make arrangements to verify that the data can be recovered
  ** before we commit */
  if( pBlob ){
    blob_reset(&cmpr);
    verify_before_commit(rid);

  }
























  return rid;
}


/*
** COMMAND:  test-content-put
**
** Extract a blob from the database and write it into a file.
*/
void test_content_put_cmd(void){







<
|







 







>
|




<
<
<
|
<







 







|








>


<




<







 







<
|
<
<






>







 







<
<
|
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


>







362
363
364
365
366
367
368

369
370
371
372
373
374
375
376
...
379
380
381
382
383
384
385
386
387
388
389
390
391



392

393
394
395
396
397
398
399
...
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427

428
429
430
431

432
433
434
435
436
437
438
...
439
440
441
442
443
444
445

446


447
448
449
450
451
452
453
454
455
456
457
458
459
460
...
476
477
478
479
480
481
482


483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
/*
** Write content into the database.  Return the record ID.  If the
** content is already in the database, just return the record ID.
**
** If srcId is specified, then pBlob is delta content from
** the srcId record.  srcId might be a phantom.
**

** If srcId is specified then the UUID is set to zUuid.  Otherwise zUuid is
** ignored.  In the future this might change such that the content
** hash is checked against zUuid to make sure it is correct.
**
** If the record already exists but is a phantom, the pBlob content
** is inserted and the phatom becomes a real record.
*/
int content_put(Blob *pBlob, const char *zUuid, int srcId){
................................................................................
  Stmt s1;
  Blob cmpr;
  Blob hash;
  int markAsUnclustered = 0;
  int isDephantomize = 0;
  
  assert( g.repositoryOpen );
  assert( pBlob!=0 );
  if( srcId==0 ){
    sha1sum_blob(pBlob, &hash);
  }else{
    blob_init(&hash, zUuid, -1);
  }



  size = blob_size(pBlob);

  db_begin_transaction();

  /* Check to see if the entry already exists and if it does whether
  ** or not the entry is a phantom
  */
  db_prepare(&s1, "SELECT rid, size FROM blob WHERE uuid=%B", &hash);
  if( db_step(&s1)==SQLITE_ROW ){
................................................................................
  }else{
    rid = 0;  /* No entry with the same UUID currently exists */
    markAsUnclustered = 1;
  }
  db_finalize(&s1);

  /* Construct a received-from ID if we do not already have one */
  if( g.rcvid==0 ){
    db_multi_exec(
       "INSERT INTO rcvfrom(uid, mtime, nonce, ipaddr)"
       "VALUES(%d, julianday('now'), %Q, %Q)",
       g.userUid, g.zNonce, g.zIpAddr
    );
    g.rcvid = db_last_insert_rowid();
  }

  blob_compress(pBlob, &cmpr);
  if( rid>0 ){
    /* We are just adding data to a phantom */

    db_prepare(&s1,
      "UPDATE blob SET rcvid=%d, size=%d, content=:data WHERE rid=%d",
       g.rcvid, size, rid
    );

    db_bind_blob(&s1, ":data", &cmpr);
    db_exec(&s1);
    db_multi_exec("DELETE FROM phantom WHERE rid=%d", rid);
    if( srcId==0 || content_is_available(srcId) ){
      isDephantomize = 1;
      content_mark_available(rid);
    }
................................................................................
  }else{
    /* We are creating a new entry */
    db_prepare(&s1,
      "INSERT INTO blob(rcvid,size,uuid,content)"
      "VALUES(%d,%d,'%b',:data)",
       g.rcvid, size, &hash
    );

    blob_compress(pBlob, &cmpr);


    db_exec(&s1);
    rid = db_last_insert_rowid();
    if( !pBlob ){
      db_multi_exec("INSERT OR IGNORE INTO phantom VALUES(%d)", rid);
    }
  }
  blob_reset(&cmpr);

  /* If the srcId is specified, then the data we just added is
  ** really a delta.  Record this fact in the delta table.
  */
  if( srcId ){
    db_multi_exec("REPLACE INTO delta(rid,srcid) VALUES(%d,%d)", rid, srcId);
  }
................................................................................
  /* Finish the transaction and cleanup */
  db_finalize(&s1);
  db_end_transaction(0);
  blob_reset(&hash);

  /* Make arrangements to verify that the data can be recovered
  ** before we commit */


  verify_before_commit(rid);
  return rid;
}

/*
** Create a new phantom with the given UUID and return its artifact ID.
*/
int content_new(const char *zUuid){
  int rid;
  static Stmt s1, s2;
  
  assert( g.repositoryOpen );
  db_begin_transaction();
  db_static_prepare(&s1,
    "INSERT INTO blob(rcvid,size,uuid,content)"
    "VALUES(0,-1,:uuid,NULL)"
  );
  db_bind_text(&s1, ":uuid", zUuid);
  db_exec(&s1);
  rid = db_last_insert_rowid();
  db_static_prepare(&s2,
    "INSERT INTO phantom VALUES(:rid)"
  );
  db_bind_int(&s2, ":rid", rid);
  db_exec(&s2);
  bag_insert(&contentCache.missing, rid);
  db_end_transaction(0);
  return rid;
}


/*
** COMMAND:  test-content-put
**
** Extract a blob from the database and write it into a file.
*/
void test_content_put_cmd(void){

Changes to src/vfile.c.

37
38
39
40
41
42
43

44
45
46
47
48
49
50
51
52







53
54
55
56
57
58
59
60
61
** user input with mixed case, use resolve_uuid().
**
** If the UUID is not found and phantomize is 1, then attempt to 
** create a phantom record.
*/
int uuid_to_rid(const char *zUuid, int phantomize){
  int rid, sz;

  char z[UUID_SIZE+1];
  
  sz = strlen(zUuid);
  if( sz!=UUID_SIZE || !validate16(zUuid, sz) ){
    return 0;
  }
  strcpy(z, zUuid);
  canonical16(z, sz);
  rid = db_int(0, "SELECT rid FROM blob WHERE uuid=%Q", z);







  if( rid==0 && phantomize ){
    rid = content_put(0, zUuid, 0);
  }
  return rid;
}

/*
** Verify that an object is not a phantom.  If the object is
** a phantom, output an error message and quick.







>








|
>
>
>
>
>
>
>

|







37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
** user input with mixed case, use resolve_uuid().
**
** If the UUID is not found and phantomize is 1, then attempt to 
** create a phantom record.
*/
int uuid_to_rid(const char *zUuid, int phantomize){
  int rid, sz;
  static Stmt q;
  char z[UUID_SIZE+1];
  
  sz = strlen(zUuid);
  if( sz!=UUID_SIZE || !validate16(zUuid, sz) ){
    return 0;
  }
  strcpy(z, zUuid);
  canonical16(z, sz);
  db_static_prepare(&q, "SELECT rid FROM blob WHERE uuid=:uuid");
  db_bind_text(&q, ":uuid", z);
  if( db_step(&q)==SQLITE_ROW ){
    rid = db_column_int(&q, 0);
  }else{
    rid = 0;
  }
  db_reset(&q);
  if( rid==0 && phantomize ){
    rid = content_new(zUuid);
  }
  return rid;
}

/*
** Verify that an object is not a phantom.  If the object is
** a phantom, output an error message and quick.

Changes to src/xfer.c.

54
55
56
57
58
59
60



61







62
63
64
65
66
67
68
69
70
...
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
** Create a phantom record if no prior record exists and
** phantomize is true.
**
** Compare to uuid_to_rid().  This routine takes a blob argument
** and does less error checking.
*/
static int rid_from_uuid(Blob *pUuid, int phantomize){



  int rid = db_int(0, "SELECT rid FROM blob WHERE uuid='%b'", pUuid);







  if( rid==0 && phantomize ){
    rid = content_put(0, blob_str(pUuid), 0);
  }
  return rid;
}

/*
** Remember that the other side of the connection already has a copy
** of the file rid.
................................................................................
      ** Always remember that the server has this file so that we do
      ** not transmit it by accident.
      */
      if( xfer.nToken==2
       && blob_eq(&xfer.aToken[0], "igot")
       && blob_is_uuid(&xfer.aToken[1])
      ){
        int rid = 0;
        if( pullFlag || cloneFlag ){
          if( !db_exists("SELECT 1 FROM blob WHERE uuid='%b' AND size>=0",
                &xfer.aToken[1]) ){
            rid = content_put(0, blob_str(&xfer.aToken[1]), 0);
            newPhantom = 1;
          }
        }
        if( rid==0 ){
          rid = rid_from_uuid(&xfer.aToken[1], 0);
        }
        remote_has(rid);
      }else
    
      
      /*   push  SERVERCODE  PRODUCTCODE
      **







>
>
>
|
>
>
>
>
>
>
>

|







 







|
|
<
<
|
|
<
<
<
<







54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
...
848
849
850
851
852
853
854
855
856


857
858




859
860
861
862
863
864
865
** Create a phantom record if no prior record exists and
** phantomize is true.
**
** Compare to uuid_to_rid().  This routine takes a blob argument
** and does less error checking.
*/
static int rid_from_uuid(Blob *pUuid, int phantomize){
  static Stmt q;
  int rid;

  db_static_prepare(&q, "SELECT rid FROM blob WHERE uuid=:uuid");
  db_bind_str(&q, ":uuid", pUuid);
  if( db_step(&q)==SQLITE_ROW ){
    rid = db_column_int(&q, 0);
  }else{
    rid = 0;
  }
  db_reset(&q);
  if( rid==0 && phantomize ){
    rid = content_new(blob_str(pUuid));
  }
  return rid;
}

/*
** Remember that the other side of the connection already has a copy
** of the file rid.
................................................................................
      ** Always remember that the server has this file so that we do
      ** not transmit it by accident.
      */
      if( xfer.nToken==2
       && blob_eq(&xfer.aToken[0], "igot")
       && blob_is_uuid(&xfer.aToken[1])
      ){
        int rid = rid_from_uuid(&xfer.aToken[1], 0);
        if( rid==0 && (pullFlag || cloneFlag) ){


          rid = content_new(blob_str(&xfer.aToken[1]));
          newPhantom = 1;




        }
        remote_has(rid);
      }else
    
      
      /*   push  SERVERCODE  PRODUCTCODE
      **