Fossil

Check-in [9e8a7d4c]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Experimental option --fast on the clone command only clones manifest artifacts and related tags. This is an experiment to see how much bandwidth and time are saved by omitting the download of file content until it is actually needed.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | fast-clone
Files: files | file ages | folders
SHA3-256:9e8a7d4ca85aaf6b1e19badbf96d802b4f75caf8b78eda41fba378e850811f94
User & Date: drh 2018-06-04 16:27:52
Context
2018-06-04
16:27
Experimental option --fast on the clone command only clones manifest artifacts and related tags. This is an experiment to see how much bandwidth and time are saved by omitting the download of file content until it is actually needed. Leaf check-in: 9e8a7d4c user: drh tags: fast-clone
14:47
Update the sync protocol document. check-in: 55cd6153 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/clone.c.

107
108
109
110
111
112
113

114
115
116
117
118
119
120
...
127
128
129
130
131
132
133

134
135
136
137
138
139
140
**
** By default, your current login name is used to create the default
** admin user. This can be overridden using the -A|--admin-user
** parameter.
**
** Options:
**    --admin-user|-A USERNAME   Make USERNAME the administrator

**    --once                     Don't remember the URI.
**    --private                  Also clone private branches
**    --ssl-identity FILENAME    Use the SSL identity if requested by the server
**    --ssh-command|-c SSH       Use SSH as the "ssh" command
**    --httpauth|-B USER:PASS    Add HTTP Basic Authorization to requests
**    -u|--unversioned           Also sync unversioned content
**    -v|--verbose               Show more statistics in output
................................................................................
  const char *zHttpAuth;      /* HTTP Authorization user:pass information */
  int nErr = 0;
  int urlFlags = URL_PROMPT_PW | URL_REMEMBER;
  int syncFlags = SYNC_CLONE;

  /* Also clone private branches */
  if( find_option("private",0,0)!=0 ) syncFlags |= SYNC_PRIVATE;

  if( find_option("once",0,0)!=0) urlFlags &= ~URL_REMEMBER;
  if( find_option("verbose","v",0)!=0) syncFlags |= SYNC_VERBOSE;
  if( find_option("unversioned","u",0)!=0 ) syncFlags |= SYNC_UNVERSIONED;
  zHttpAuth = find_option("httpauth","B",1);
  zDefaultUser = find_option("admin-user","A",1);
  clone_ssh_find_options();
  url_proxy_options();







>







 







>







107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
...
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
**
** By default, your current login name is used to create the default
** admin user. This can be overridden using the -A|--admin-user
** parameter.
**
** Options:
**    --admin-user|-A USERNAME   Make USERNAME the administrator
**    --fast                     Minimal clone - most content left on server
**    --once                     Don't remember the URI.
**    --private                  Also clone private branches
**    --ssl-identity FILENAME    Use the SSL identity if requested by the server
**    --ssh-command|-c SSH       Use SSH as the "ssh" command
**    --httpauth|-B USER:PASS    Add HTTP Basic Authorization to requests
**    -u|--unversioned           Also sync unversioned content
**    -v|--verbose               Show more statistics in output
................................................................................
  const char *zHttpAuth;      /* HTTP Authorization user:pass information */
  int nErr = 0;
  int urlFlags = URL_PROMPT_PW | URL_REMEMBER;
  int syncFlags = SYNC_CLONE;

  /* Also clone private branches */
  if( find_option("private",0,0)!=0 ) syncFlags |= SYNC_PRIVATE;
  if( find_option("fast",0,0)!=0 ) syncFlags |= SYNC_ONLY_CKIN;
  if( find_option("once",0,0)!=0) urlFlags &= ~URL_REMEMBER;
  if( find_option("verbose","v",0)!=0) syncFlags |= SYNC_VERBOSE;
  if( find_option("unversioned","u",0)!=0 ) syncFlags |= SYNC_UNVERSIONED;
  zHttpAuth = find_option("httpauth","B",1);
  zDefaultUser = find_option("admin-user","A",1);
  clone_ssh_find_options();
  url_proxy_options();

Changes to src/xfer.c.

47
48
49
50
51
52
53


54
55

56
57
58
59
60
61
62
....
1350
1351
1352
1353
1354
1355
1356
1357



1358
1359
1360
1361
1362
1363
1364
....
1367
1368
1369
1370
1371
1372
1373


1374
1375
1376
1377
1378
1379
1380
1381
....
1514
1515
1516
1517
1518
1519
1520





























1521
1522
1523
1524
1525
1526
1527
....
1652
1653
1654
1655
1656
1657
1658

1659
1660
1661
1662
1663
1664
1665
....
1743
1744
1745
1746
1747
1748
1749
1750





1751
1752
1753
1754
1755
1756
1757
....
1820
1821
1822
1823
1824
1825
1826

1827
1828
1829
1830
1831
1832
1833
1834
....
2167
2168
2169
2170
2171
2172
2173
2174

2175



2176
2177
2178
2179
2180
2181
2182
  int nFileRcvd;      /* Number of files received */
  int nDeltaRcvd;     /* Number of deltas received */
  int nDanglingFile;  /* Number of dangling deltas received */
  int mxSend;         /* Stop sending "file" when pOut reaches this size */
  int resync;         /* Send igot cards for all holdings */
  u8 syncPrivate;     /* True to enable syncing private content */
  u8 nextIsPrivate;   /* If true, next "file" received is a private */


  u32 clientVersion;  /* Version of the client software */
  time_t maxTime;     /* Time when this transfer should be finished */

};


/*
** The input blob contains an artifact.  Convert it into a record ID.
** Create a phantom record if no prior record exists and
** phantomize is true.
................................................................................
      if( !g.perm.Clone ){
        cgi_reset_content();
        @ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
        @ error not\sauthorized\sto\sclone
        nErr++;
        break;
      }
      if( db_get_boolean("uv-sync",0) && !uvCatalogSent ){



        @ pragma uv-pull-only
        send_unversioned_catalog(&xfer);
        uvCatalogSent = 1;
      }
      if( xfer.nToken==3
       && blob_is_int(&xfer.aToken[1], &iVers)
       && iVers>=2
................................................................................
        if( iVers>=3 ){
          cgi_set_content_type("application/x-fossil-uncompressed");
        }
        blob_is_int(&xfer.aToken[2], &seqno);
        max = db_int(0, "SELECT max(rid) FROM blob");
        while( xfer.mxSend>blob_size(xfer.pOut) && seqno<=max){
          if( time(NULL) >= xfer.maxTime ) break;


          if( iVers>=3 ){
            send_compressed_file(&xfer, seqno);
          }else{
            send_file(&xfer, seqno, 0, 1);
          }
          seqno++;
        }
        if( seqno>max ) seqno = 0;
................................................................................
      /*   pragma send-catalog
      **
      ** Send igot cards for all known artifacts.
      */
      if( blob_eq(&xfer.aToken[1], "send-catalog") ){
        xfer.resync = 0x7fffffff;
      }






























      /*   pragma client-version VERSION
      **
      ** Let the server know what version of Fossil is running on the client.
      */
      if( xfer.nToken>=3 && blob_eq(&xfer.aToken[1], "client-version") ){
        xfer.clientVersion = atoi(blob_str(&xfer.aToken[2]));
................................................................................
#define SYNC_VERBOSE        0x0010    /* Extra diagnostics */
#define SYNC_RESYNC         0x0020    /* --verily */
#define SYNC_UNVERSIONED    0x0040    /* Sync unversioned content */
#define SYNC_UV_REVERT      0x0080    /* Copy server unversioned to client */
#define SYNC_FROMPARENT     0x0100    /* Pull from the parent project */
#define SYNC_UV_TRACE       0x0200    /* Describe UV activities */
#define SYNC_UV_DRYRUN      0x0400    /* Do not actually exchange files */

#endif

/*
** Floating-point absolute value
*/
static double fossil_fabs(double x){
  return x>0.0 ? x : -x;
................................................................................
  blob_zero(&xfer.err);
  blob_zero(&xfer.line);
  origConfigRcvMask = 0;


  /* Send the send-private pragma if we are trying to sync private data */
  if( syncFlags & SYNC_PRIVATE ){
    blob_append(&send, "pragma send-private\n", -1);





  }

  /* When syncing unversioned files, create a TEMP table in which to store
  ** the names of files that need to be sent from client to server.
  **
  ** The initial assumption is that all unversioned files need to be sent
  ** to the other side.  But "uvigot" cards received back from the remote
................................................................................
      blob_appendf(&send, "cookie %s\n", zCookie);
    }

    /* Generate gimme cards for phantoms and leaf cards
    ** for all leaves.
    */
    if( (syncFlags & SYNC_PULL)!=0

     || ((syncFlags & SYNC_CLONE)!=0 && cloneSeqno==1)
    ){
      request_phantoms(&xfer, mxPhantomReq);
    }
    if( syncFlags & SYNC_PUSH ){
      send_unsent(&xfer);
      nCardSent += send_unclustered(&xfer);
      if( syncFlags & SYNC_PRIVATE ) send_private(&xfer);
................................................................................
       && (syncFlags & SYNC_CLONE)!=0
       && blob_is_hname(&xfer.aToken[2])
      ){
        if( zPCode==0 ){
          zPCode = mprintf("%b", &xfer.aToken[2]);
          db_set("project-code", zPCode, 0);
        }
        if( cloneSeqno>0 ) blob_appendf(&send, "clone 3 %d\n", cloneSeqno);

        nCardSent++;



      }else

      /*   config NAME SIZE \n CONTENT
      **
      ** Receive a configuration value from the server.
      **
      ** The received configuration setting is silently ignored if it was







>
>


>







 







|
>
>
>







 







>
>
|







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>







 







|
>
>
>
>
>







 







>
|







 







|
>
|
>
>
>







47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
....
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
....
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
....
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
....
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
....
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
....
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
....
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
  int nFileRcvd;      /* Number of files received */
  int nDeltaRcvd;     /* Number of deltas received */
  int nDanglingFile;  /* Number of dangling deltas received */
  int mxSend;         /* Stop sending "file" when pOut reaches this size */
  int resync;         /* Send igot cards for all holdings */
  u8 syncPrivate;     /* True to enable syncing private content */
  u8 nextIsPrivate;   /* If true, next "file" received is a private */
  u8 ckinOnly;        /* Clone only check-ins and related artifacts */
  u8 useOkToSend;     /* Omit artifacts found in okToSend */
  u32 clientVersion;  /* Version of the client software */
  time_t maxTime;     /* Time when this transfer should be finished */
  Bag okToSend;       /* IDs of BLOBs acceptable to send */
};


/*
** The input blob contains an artifact.  Convert it into a record ID.
** Create a phantom record if no prior record exists and
** phantomize is true.
................................................................................
      if( !g.perm.Clone ){
        cgi_reset_content();
        @ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
        @ error not\sauthorized\sto\sclone
        nErr++;
        break;
      }
      if( db_get_boolean("uv-sync",0)
       && !uvCatalogSent
       && !xfer.ckinOnly
      ){
        @ pragma uv-pull-only
        send_unversioned_catalog(&xfer);
        uvCatalogSent = 1;
      }
      if( xfer.nToken==3
       && blob_is_int(&xfer.aToken[1], &iVers)
       && iVers>=2
................................................................................
        if( iVers>=3 ){
          cgi_set_content_type("application/x-fossil-uncompressed");
        }
        blob_is_int(&xfer.aToken[2], &seqno);
        max = db_int(0, "SELECT max(rid) FROM blob");
        while( xfer.mxSend>blob_size(xfer.pOut) && seqno<=max){
          if( time(NULL) >= xfer.maxTime ) break;
          if( xfer.useOkToSend && bag_find(&xfer.okToSend, seqno)==0 ){
            /* no-op */
          }else if( iVers>=3 ){
            send_compressed_file(&xfer, seqno);
          }else{
            send_file(&xfer, seqno, 0, 1);
          }
          seqno++;
        }
        if( seqno>max ) seqno = 0;
................................................................................
      /*   pragma send-catalog
      **
      ** Send igot cards for all known artifacts.
      */
      if( blob_eq(&xfer.aToken[1], "send-catalog") ){
        xfer.resync = 0x7fffffff;
      }

      /*   pragma ckin-only
      **
      ** Only send manifests and related tag artifacts.  Omit
      ** file content, wiki, and ticket artifacts.
      */
      if( blob_eq(&xfer.aToken[1], "ckin-only") ){
        Stmt q;
        if( !xfer.useOkToSend ){
          xfer.useOkToSend = 1;
          bag_init(&xfer.okToSend);
        }
        db_prepare(&q, "SELECT objid FROM event WHERE type='ci'");
        while( db_step(&q)==SQLITE_ROW ){
          bag_insert(&xfer.okToSend, db_column_int(&q, 0));
        }
        db_finalize(&q);
        db_prepare(&q,
          "SELECT srcid FROM tagxref"
          " WHERE srcid!=0"
          "   AND srcid!=rid"
          "   AND rid IN (SELECT objid FROM event WHERE type='ci')"
        );
        while( db_step(&q)==SQLITE_ROW ){
          bag_insert(&xfer.okToSend, db_column_int(&q, 0));
        }
        db_finalize(&q);
        xfer.ckinOnly = 1;
      }

      /*   pragma client-version VERSION
      **
      ** Let the server know what version of Fossil is running on the client.
      */
      if( xfer.nToken>=3 && blob_eq(&xfer.aToken[1], "client-version") ){
        xfer.clientVersion = atoi(blob_str(&xfer.aToken[2]));
................................................................................
#define SYNC_VERBOSE        0x0010    /* Extra diagnostics */
#define SYNC_RESYNC         0x0020    /* --verily */
#define SYNC_UNVERSIONED    0x0040    /* Sync unversioned content */
#define SYNC_UV_REVERT      0x0080    /* Copy server unversioned to client */
#define SYNC_FROMPARENT     0x0100    /* Pull from the parent project */
#define SYNC_UV_TRACE       0x0200    /* Describe UV activities */
#define SYNC_UV_DRYRUN      0x0400    /* Do not actually exchange files */
#define SYNC_ONLY_CKIN      0x0800    /* Only manifasts & related tags */
#endif

/*
** Floating-point absolute value
*/
static double fossil_fabs(double x){
  return x>0.0 ? x : -x;
................................................................................
  blob_zero(&xfer.err);
  blob_zero(&xfer.line);
  origConfigRcvMask = 0;


  /* Send the send-private pragma if we are trying to sync private data */
  if( syncFlags & SYNC_PRIVATE ){
    blob_append(&send, "pragma send-private\n", -1); nCardSent++;
  }

  /* Let the server know if we are only interested in check-ins */
  if( syncFlags & SYNC_ONLY_CKIN ){
    blob_appendf(&send, "pragma ckin-only\n"); nCardSent++;
  }

  /* When syncing unversioned files, create a TEMP table in which to store
  ** the names of files that need to be sent from client to server.
  **
  ** The initial assumption is that all unversioned files need to be sent
  ** to the other side.  But "uvigot" cards received back from the remote
................................................................................
      blob_appendf(&send, "cookie %s\n", zCookie);
    }

    /* Generate gimme cards for phantoms and leaf cards
    ** for all leaves.
    */
    if( (syncFlags & SYNC_PULL)!=0
     || ((syncFlags & (SYNC_CLONE|SYNC_ONLY_CKIN))==SYNC_CLONE
           && cloneSeqno==1)
    ){
      request_phantoms(&xfer, mxPhantomReq);
    }
    if( syncFlags & SYNC_PUSH ){
      send_unsent(&xfer);
      nCardSent += send_unclustered(&xfer);
      if( syncFlags & SYNC_PRIVATE ) send_private(&xfer);
................................................................................
       && (syncFlags & SYNC_CLONE)!=0
       && blob_is_hname(&xfer.aToken[2])
      ){
        if( zPCode==0 ){
          zPCode = mprintf("%b", &xfer.aToken[2]);
          db_set("project-code", zPCode, 0);
        }
        if( cloneSeqno>0 ){
          if( syncFlags & SYNC_ONLY_CKIN ){
            blob_appendf(&send, "pragma ckin-only\n"); nCardSent++;
          }
          blob_appendf(&send, "clone 3 %d\n", cloneSeqno); nCardSent++;
        }
      }else

      /*   config NAME SIZE \n CONTENT
      **
      ** Receive a configuration value from the server.
      **
      ** The received configuration setting is silently ignored if it was