Fossil

Check-in [faa73ae0]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Limit the amount of unversioned content sent with each HTTP request. Also: fix up source code lines in xfer.c to be less than 80 characters long.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | unversioned-files
Files: files | file ages | folders
SHA1:faa73ae0c4a82af20e48134750dddf21ae37859e
User & Date: drh 2016-08-17 13:37:43
Context
2016-08-17
14:40
Update the sync protocol documentation to describe the unversioned file sync. check-in: d7809493 user: drh tags: unversioned-files
13:37
Limit the amount of unversioned content sent with each HTTP request. Also: fix up source code lines in xfer.c to be less than 80 characters long. check-in: faa73ae0 user: drh tags: unversioned-files
11:53
The admin and setup users automatically get the write-unversioned ('y') permission. check-in: 0480400c user: drh tags: unversioned-files
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/xfer.c.

     1      1   /*
     2      2   ** Copyright (c) 2007 D. Richard Hipp
     3      3   **
     4      4   ** This program is free software; you can redistribute it and/or
     5      5   ** modify it under the terms of the Simplified BSD License (also
     6      6   ** known as the "2-Clause License" or "FreeBSD License".)
     7         -
            7  +**
     8      8   ** This program is distributed in the hope that it will be useful,
     9      9   ** but without any warranty; without even the implied warranty of
    10     10   ** merchantability or fitness for a particular purpose.
    11     11   **
    12     12   ** Author contact information:
    13     13   **   drh@hwaci.com
    14     14   **   http://www.hwaci.com/drh/
................................................................................
   303    303   **
   304    304   ** If the 0x0004 bit of FLAGS is set, that means the CONTENT is omitted.
   305    305   ** The sender might have omitted the content because it is too big to
   306    306   ** transmit, or because it is unchanged and this record exists purely
   307    307   ** to update the MTIME.
   308    308   */
   309    309   static void xfer_accept_unversioned_file(Xfer *pXfer, int isWriter){
   310         -  sqlite3_int64 mtime;      /* The MTIME */
   311         -  Blob *pHash;              /* The HASH value */
   312         -  int sz;                   /* The SIZE */
   313         -  int flags;                /* The FLAGS */
   314         -  Blob content;             /* The CONTENT */
   315         -  Blob hash;                /* Hash computed from CONTENT to compare with HASH */
   316         -  Blob x;                   /* Compressed content */
   317         -  Stmt q;                   /* SQL statements for comparison and insert */
   318         -  int isDelete;             /* HASH is "-" indicating this is a delete operation */
   319         -  int nullContent;          /* True of CONTENT is NULL */
   320         -  int iStatus;              /* Result from unversioned_status() */
          310  +  sqlite3_int64 mtime;    /* The MTIME */
          311  +  Blob *pHash;            /* The HASH value */
          312  +  int sz;                 /* The SIZE */
          313  +  int flags;              /* The FLAGS */
          314  +  Blob content;           /* The CONTENT */
          315  +  Blob hash;              /* Hash computed from CONTENT to compare with HASH */
          316  +  Blob x;                 /* Compressed content */
          317  +  Stmt q;                 /* SQL statements for comparison and insert */
          318  +  int isDelete;           /* HASH is "-" indicating this is a delete */
          319  +  int nullContent;        /* True of CONTENT is NULL */
          320  +  int iStatus;            /* Result from unversioned_status() */
   321    321   
   322    322     pHash = &pXfer->aToken[3];
   323    323     if( pXfer->nToken==5
   324    324      || !blob_is_filename(&pXfer->aToken[1])
   325    325      || !blob_is_int64(&pXfer->aToken[2], &mtime)
   326    326      || (!blob_eq(pHash,"-") && !blob_is_uuid(pHash))
   327    327      || !blob_is_int(&pXfer->aToken[4], &sz)
................................................................................
   348    348     /* The isWriter flag must be true in order to land the new file */
   349    349     if( !isWriter ) goto end_accept_unversioned_file;
   350    350   
   351    351     /* Check to see if current content really should be overwritten.  Ideally,
   352    352     ** a uvfile card should never have been sent unless the overwrite should
   353    353     ** occur.  But do not trust the sender.  Double-check. 
   354    354     */
   355         -  iStatus = unversioned_status(blob_str(&pXfer->aToken[1]), mtime, blob_str(pHash));
          355  +  iStatus = unversioned_status(blob_str(&pXfer->aToken[1]), mtime,
          356  +                               blob_str(pHash));
   356    357     if( iStatus>=3 ) goto end_accept_unversioned_file;
   357    358     
   358    359     /* Store the content */
   359    360     isDelete = blob_eq(pHash, "-");
   360    361     if( isDelete ){
   361    362       db_prepare(&q,
   362    363         "UPDATE unversioned"
   363         -      "   SET rcvid=:rcvid, mtime=:mtime, hash=NULL, sz=0, encoding=0, content=NULL"
          364  +      "   SET rcvid=:rcvid, mtime=:mtime, hash=NULL,"
          365  +      "       sz=0, encoding=0, content=NULL"
   364    366         " WHERE name=:name"
   365    367       );
   366    368       db_bind_int(&q, ":rcvid", g.rcvid);
   367    369     }else if( iStatus==4 ){
   368    370       db_prepare(&q, "UPDATE unversioned SET mtime=:mtime WHERE name=:name");
   369    371     }else{
   370    372       db_prepare(&q,
   371         -      "REPLACE INTO unversioned(name, rcvid, mtime, hash, sz, encoding, content)"
          373  +      "REPLACE INTO unversioned(name,rcvid,mtime,hash,sz,encoding,content)"
   372    374         " VALUES(:name,:rcvid,:mtime,:hash,:sz,:encoding,:content)"
   373    375       );
   374    376       db_bind_int(&q, ":rcvid", g.rcvid);
   375    377       db_bind_text(&q, ":hash", blob_str(pHash));
   376    378       db_bind_int(&q, ":sz", blob_size(&content));
   377    379       if( !nullContent ){
   378    380         blob_compress(&content, &x);
................................................................................
   643    645   
   644    646   /*
   645    647   ** Send the unversioned file identified by zName by generating the
   646    648   ** appropriate "uvfile" card.
   647    649   **
   648    650   **     uvfile NAME MTIME HASH SIZE FLAGS \n CONTENT
   649    651   **
   650         -** If the noContent flag is set, omit the CONTENT and set the 0x0004 flag in FLAGS.
          652  +** If the noContent flag is set, omit the CONTENT and set the 0x0004
          653  +** flag in FLAGS.
   651    654   */
   652         -static void send_unversioned_file(Xfer *pXfer, const char *zName, int noContent){
          655  +static void send_unversioned_file(
          656  +  Xfer *pXfer,            /* Transfer context */
          657  +  const char *zName,      /* Name of unversioned file to be sent */
          658  +  int noContent           /* True to omit the content */
          659  +){
   653    660     Stmt q1;
   654    661   
          662  +  if( blob_size(pXfer->pOut)>=pXfer->mxSend ) noContent = 1;
   655    663     if( noContent ){
   656    664       db_prepare(&q1,
   657    665         "SELECT mtime, hash, encoding, sz FROM unversioned WHERE name=%Q",
   658    666         zName
   659    667       );
   660    668     }else{
   661    669       db_prepare(&q1,
   662         -      "SELECT mtime, hash, encoding, sz, content FROM unversioned WHERE name=%Q",
          670  +      "SELECT mtime, hash, encoding, sz, content FROM unversioned"
          671  +      " WHERE name=%Q",
   663    672         zName
   664    673       );
   665    674     }
   666    675     if( db_step(&q1)==SQLITE_ROW ){
   667    676       sqlite3_int64 mtime = db_column_int64(&q1, 0);
   668    677       const char *zHash = db_column_text(&q1, 1);
   669         -    blob_appendf(pXfer->pOut, "uvfile %s %lld", zName, mtime);
   670         -    if( zHash==0 ){
   671         -      blob_append(pXfer->pOut, " - 0 1\n", -1);
   672         -    }else if( noContent ){
   673         -      blob_appendf(pXfer->pOut, " %s %d 4\n", zHash, db_column_int(&q1,3));
          678  +    if( blob_size(pXfer->pOut)>=pXfer->mxSend ){
          679  +      /* If we have already reached the send size limit, send a (short)
          680  +      ** uvigot card rather than a uvfile card.  This only happens on the
          681  +      ** server side.  The uvigot card will provoke the client to resend
          682  +      ** another uvgimme on the next cycle. */
          683  +      blob_appendf(pXfer->pOut, "uvigot %s %lld %s %d\n",
          684  +                   zName, mtime, zHash, db_column_int(&q1,3));
   674    685       }else{
   675         -      Blob content;
   676         -      blob_init(&content, 0, 0);
   677         -      db_column_blob(&q1, 4, &content);
   678         -      if( db_column_int(&q1, 2) ){
   679         -        blob_uncompress(&content, &content);
          686  +      blob_appendf(pXfer->pOut, "uvfile %s %lld", zName, mtime);
          687  +      if( zHash==0 ){
          688  +        blob_append(pXfer->pOut, " - 0 1\n", -1);
          689  +      }else if( noContent ){
          690  +        blob_appendf(pXfer->pOut, " %s %d 4\n", zHash, db_column_int(&q1,3));
          691  +      }else{
          692  +        Blob content;
          693  +        blob_init(&content, 0, 0);
          694  +        db_column_blob(&q1, 4, &content);
          695  +        if( db_column_int(&q1, 2) ){
          696  +          blob_uncompress(&content, &content);
          697  +        }
          698  +        blob_appendf(pXfer->pOut, " %s %d 0\n", zHash, blob_size(&content));
          699  +        blob_append(pXfer->pOut, blob_buffer(&content), blob_size(&content));
          700  +        blob_reset(&content);
   680    701         }
   681         -      blob_appendf(pXfer->pOut, " %s %d 0\n", zHash, blob_size(&content));
   682         -      blob_append(pXfer->pOut, blob_buffer(&content), blob_size(&content));
   683         -#if 0
   684         -      if( blob_buffer(pXfer->pOut)[blob_size(pXfer->pOut)-1]!='\n' ){
   685         -        blob_append(pXfer->pOut, "\n", 1);
   686         -      }
   687         -#endif
   688         -      blob_reset(&content);
   689    702       }
   690    703     }
   691    704     db_finalize(&q1);
   692    705   }
   693    706   
   694    707   /*
   695    708   ** Send a gimme message for every phantom.
................................................................................
   755    768   */
   756    769   int check_login(Blob *pLogin, Blob *pNonce, Blob *pSig){
   757    770     Stmt q;
   758    771     int rc = -1;
   759    772     char *zLogin = blob_terminate(pLogin);
   760    773     defossilize(zLogin);
   761    774   
   762         -  if( fossil_strcmp(zLogin, "nobody")==0 || fossil_strcmp(zLogin,"anonymous")==0 ){
          775  +  if( fossil_strcmp(zLogin, "nobody")==0
          776  +   || fossil_strcmp(zLogin,"anonymous")==0
          777  +  ){
   763    778       return 0;   /* Anybody is allowed to sync as "nobody" or "anonymous" */
   764    779     }
   765    780     if( fossil_strcmp(P("REMOTE_USER"), zLogin)==0
   766    781         && db_get_boolean("remote_user_ok",0) ){
   767    782       return 0;   /* Accept Basic Authorization */
   768    783     }
   769    784     db_prepare(&q,
................................................................................
  1515   1530   
  1516   1531         /*   pragma uv-hash HASH
  1517   1532         **
  1518   1533         ** The client wants to make sure that unversioned files are all synced.
  1519   1534         ** If the HASH does not match, send a complete catalog of
  1520   1535         ** "uvigot" cards.
  1521   1536         */
  1522         -      if( blob_eq(&xfer.aToken[1], "uv-hash") && blob_is_uuid(&xfer.aToken[2]) ){
         1537  +      if( blob_eq(&xfer.aToken[1], "uv-hash")
         1538  +       && blob_is_uuid(&xfer.aToken[2])
         1539  +      ){
  1523   1540           if( g.perm.Read && g.perm.WrUnver ){
  1524   1541             @ pragma uv-push-ok
  1525   1542             send_unversioned_catalog(&xfer);
  1526   1543           }else if( g.perm.Read ){
  1527   1544             @ pragma uv-pull-only
  1528   1545             send_unversioned_catalog(&xfer);
  1529   1546           }
................................................................................
  1682   1699     int nRoundtrip= 0;      /* Number of HTTP requests */
  1683   1700     int nArtifactSent = 0;  /* Total artifacts sent */
  1684   1701     int nArtifactRcvd = 0;  /* Total artifacts received */
  1685   1702     const char *zOpType = 0;/* Push, Pull, Sync, Clone */
  1686   1703     double rSkew = 0.0;     /* Maximum time skew */
  1687   1704     int uvHashSent = 0;     /* The "pragma uv-hash" message has been sent */
  1688   1705     int uvStatus = 0;       /* 0: no I/O.  1: pull-only  2: push-and-pull */
  1689         -  int uvDoPush = 0;       /* If true, generate uvfile messages to send to server */
         1706  +  int uvDoPush = 0;       /* Generate uvfile messages to send to server */
         1707  +  int nUvGimmeSent = 0;   /* Number of uvgimme cards sent on this cycle */
         1708  +  int nUvFileRcvd = 0;    /* Number of uvfile cards received on this cycle */
  1690   1709     sqlite3_int64 mtime;    /* Modification time on a UV file */
  1691   1710   
  1692   1711     if( db_get_boolean("dont-push", 0) ) syncFlags &= ~SYNC_PUSH;
  1693   1712     if( (syncFlags & (SYNC_PUSH|SYNC_PULL|SYNC_CLONE|SYNC_UNVERSIONED))==0
  1694   1713        && configRcvMask==0 && configSendMask==0 ) return 0;
  1695   1714     if( syncFlags & SYNC_FROMPARENT ){
  1696   1715       configRcvMask = 0;
................................................................................
  1859   1878       }
  1860   1879   
  1861   1880       /* Send unversioned files present here on the client but missing or
  1862   1881       ** obsolete on the server.
  1863   1882       */
  1864   1883       if( uvDoPush ){
  1865   1884         Stmt uvq;
         1885  +      int rc = SQLITE_OK;
  1866   1886         assert( (syncFlags & SYNC_UNVERSIONED)!=0 );
  1867   1887         assert( uvStatus==2 );
  1868   1888         db_prepare(&uvq, "SELECT name, mtimeOnly FROM uv_tosend");
  1869         -      while( db_step(&uvq)==SQLITE_ROW ){
  1870         -        send_unversioned_file(&xfer, db_column_text(&uvq,0), db_column_int(&uvq,1));
         1889  +      while( (rc = db_step(&uvq))==SQLITE_ROW ){
         1890  +        const char *zName = db_column_text(&uvq, 0);
         1891  +        send_unversioned_file(&xfer, zName, db_column_int(&uvq,1));
  1871   1892           nCardSent++;
  1872   1893           nArtifactSent++;
         1894  +        db_multi_exec("DELETE FROM uv_tosend WHERE name=%Q", zName);
         1895  +        if( blob_size(xfer.pOut)>xfer.mxSend ) break;
  1873   1896         }
  1874   1897         db_finalize(&uvq);
  1875         -      uvDoPush = 0;
         1898  +      if( rc==SQLITE_DONE ) uvDoPush = 0;
  1876   1899       }
  1877   1900   
  1878   1901       /* Append randomness to the end of the message.  This makes all
  1879   1902       ** messages unique so that that the login-card nonce will always
  1880   1903       ** be unique.
  1881   1904       */
  1882   1905       zRandomness = db_text(0, "SELECT hex(randomblob(20))");
................................................................................
  1930   1953         nCardSent++;
  1931   1954       }
  1932   1955       if( syncFlags & SYNC_PUSH ){
  1933   1956         blob_appendf(&send, "push %s %s\n", zSCode, zPCode);
  1934   1957         nCardSent++;
  1935   1958       }
  1936   1959       go = 0;
         1960  +    nUvGimmeSent = 0;
         1961  +    nUvFileRcvd = 0;
  1937   1962   
  1938   1963       /* Process the reply that came back from the server */
  1939   1964       while( blob_line(&recv, &xfer.line) ){
  1940   1965         if( blob_buffer(&xfer.line)[0]=='#' ){
  1941   1966           const char *zLine = blob_buffer(&xfer.line);
  1942   1967           if( memcmp(zLine, "# timestamp ", 12)==0 ){
  1943   1968             char zTime[20];
  1944   1969             double rDiff;
  1945   1970             sqlite3_snprintf(sizeof(zTime), zTime, "%.19s", &zLine[12]);
  1946   1971             rDiff = db_double(9e99, "SELECT julianday('%q') - %.17g",
  1947   1972                               zTime, rArrivalTime);
  1948   1973             if( rDiff>9e98 || rDiff<-9e98 ) rDiff = 0.0;
  1949         -          if( rDiff*24.0*3600.0 >= -(blob_size(&recv)/5000.0 + 20) ) rDiff = 0.0;
         1974  +          if( rDiff*24.0*3600.0 >= -(blob_size(&recv)/5000.0 + 20) ){
         1975  +            rDiff = 0.0;
         1976  +          }
  1950   1977             if( fossil_fabs(rDiff)>fossil_fabs(rSkew) ) rSkew = rDiff;
  1951   1978           }
  1952   1979           nCardRcvd++;
  1953   1980           continue;
  1954   1981         }
  1955   1982         xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken, count(xfer.aToken));
  1956   1983         nCardRcvd++;
................................................................................
  1986   2013         /*   uvfile NAME MTIME HASH SIZE FLAGS \n CONTENT
  1987   2014         **
  1988   2015         ** Accept an unversioned file from the client.
  1989   2016         */
  1990   2017         if( blob_eq(&xfer.aToken[0], "uvfile") ){
  1991   2018           xfer_accept_unversioned_file(&xfer, 1);
  1992   2019           nArtifactRcvd++;
         2020  +        nUvFileRcvd++;
  1993   2021         }else
  1994   2022   
  1995   2023         /*   gimme UUID
  1996   2024         **
  1997   2025         ** Server is requesting a file.  If the file is a manifest, assume
  1998   2026         ** that the server will also want to know all of the content files
  1999   2027         ** associated with the manifest and send those too.
................................................................................
  2041   2069         /*   uvigot NAME MTIME HASH SIZE
  2042   2070         **
  2043   2071         ** Server announces that it has a particular unversioned file.  The
  2044   2072         ** server will only send this card if the client had previously sent
  2045   2073         ** a "pragma uv-hash" card with a hash that does not match.
  2046   2074         **
  2047   2075         ** If the identified file needs to be transferred, then setup for the
  2048         -      ** transfer.  Generate a "uvgimme" card in the reply if the server version
  2049         -      ** is newer than the client.  Generate a "uvfile" card if the client version
  2050         -      ** is newer than the server.  If HASH is "-" (indicating that the file has
  2051         -      ** been deleted) and MTIME is newer, then do the deletion.
         2076  +      ** transfer.  Generate a "uvgimme" card in the reply if the server
         2077  +      ** version is newer than the client.  Generate a "uvfile" card if
         2078  +      ** the client version is newer than the server.  If HASH is "-"
         2079  +      ** (indicating that the file has been deleted) and MTIME is newer,
         2080  +      ** then do the deletion.
  2052   2081         */
  2053   2082         if( xfer.nToken==5
  2054   2083          && blob_eq(&xfer.aToken[0], "uvigot")
  2055   2084          && blob_is_filename(&xfer.aToken[1])
  2056   2085          && blob_is_int64(&xfer.aToken[2], &mtime)
  2057   2086          && blob_is_int(&xfer.aToken[4], &size)
  2058   2087          && (blob_eq(&xfer.aToken[3],"-") || blob_is_uuid(&xfer.aToken[3]))
................................................................................
  2062   2091           int iStatus;
  2063   2092           if( uvStatus==0 ) uvStatus = 2;
  2064   2093           iStatus = unversioned_status(zName, mtime, zHash);
  2065   2094           if( iStatus<=1 ){
  2066   2095             if( zHash[0]!='-' ){
  2067   2096               blob_appendf(xfer.pOut, "uvgimme %s\n", zName);
  2068   2097               nCardSent++;
         2098  +            nUvGimmeSent++;
  2069   2099             }else if( iStatus==1 ){
  2070   2100               db_multi_exec(
  2071   2101                  "UPDATE unversioned"
  2072   2102                  "   SET mtime=%lld, hash=NULL, sz=0, encoding=0, content=NULL"
  2073   2103                  " WHERE name=%Q", mtime, zName
  2074   2104               );
  2075   2105               db_unset("uv-hash", 0);
................................................................................
  2079   2109               "UPDATE unversioned SET mtime=%lld WHERE name=%Q", mtime, zName
  2080   2110             );
  2081   2111             db_unset("uv-hash", 0);
  2082   2112           }
  2083   2113           if( iStatus<=3 ){
  2084   2114             db_multi_exec("DELETE FROM uv_tosend WHERE name=%Q", zName);
  2085   2115           }else if( iStatus==4 ){
  2086         -          db_multi_exec("UPDATE uv_tosend SET mtimeOnly=1 WHERE name=%Q", zName);
         2116  +          db_multi_exec("UPDATE uv_tosend SET mtimeOnly=1 WHERE name=%Q",zName);
  2087   2117           }else if( iStatus==5 ){
  2088         -          db_multi_exec("REPLACE INTO uv_tosend(name,mtimeOnly) VALUES(%Q,0)", zName);
         2118  +          db_multi_exec("REPLACE INTO uv_tosend(name,mtimeOnly) VALUES(%Q,0)",
         2119  +                        zName);
  2089   2120           }
  2090   2121         }else
  2091   2122   
  2092   2123         /*   push  SERVERCODE  PRODUCTCODE
  2093   2124         **
  2094   2125         ** Should only happen in response to a clone.  This message tells
  2095   2126         ** the client what product to use for the new database.
................................................................................
  2170   2201         **
  2171   2202         ** If the "login failed" message is seen, clear the sync password prior
  2172   2203         ** to the next cycle.
  2173   2204         */
  2174   2205         if( blob_eq(&xfer.aToken[0],"message") && xfer.nToken==2 ){
  2175   2206           char *zMsg = blob_terminate(&xfer.aToken[1]);
  2176   2207           defossilize(zMsg);
  2177         -        if( (syncFlags & SYNC_PUSH) && zMsg && sqlite3_strglob("pull only *", zMsg)==0 ){
         2208  +        if( (syncFlags & SYNC_PUSH) && zMsg
         2209  +            && sqlite3_strglob("pull only *", zMsg)==0 ){
  2178   2210             syncFlags &= ~SYNC_PUSH;
  2179   2211             zMsg = 0;
  2180   2212           }
  2181   2213           if( zMsg && zMsg[0] ){
  2182   2214             fossil_force_newline();
  2183   2215             fossil_print("Server says: %s\n", zMsg);
  2184   2216           }
................................................................................
  2300   2332   
  2301   2333       /* If we have one or more files queued to send, then go
  2302   2334       ** another round
  2303   2335       */
  2304   2336       if( xfer.nFileSent+xfer.nDeltaSent>0 || uvDoPush ){
  2305   2337         go = 1;
  2306   2338       }
         2339  +
         2340  +    /* Continue looping as long as new uvfile cards are being received
         2341  +    ** and uvgimme cards are being sent. */
         2342  +    if( nUvGimmeSent>0 && nUvFileRcvd>0 ) go = 1;
  2307   2343   
  2308   2344       /* If this is a clone, the go at least two rounds */
  2309   2345       if( (syncFlags & SYNC_CLONE)!=0 && nCycle==1 ) go = 1;
  2310   2346   
  2311   2347       /* Stop the cycle if the server sends a "clone_seqno 0" card and
  2312   2348       ** we have gone at least two rounds.  Always go at least two rounds
  2313   2349       ** on a clone in order to be sure to retrieve the configuration