Fossil

**
** Author contact information:
**   drh@hwaci.com
**   http://www.hwaci.com/drh/
**
*******************************************************************************
**
** This file contains code used to generate ZIP and SQLAR archives.
*/
#include "config.h"
#include <assert.h>
#if defined(FOSSIL_ENABLE_MINIZ)
#  define MINIZ_HEADER_FILE_ONLY
#  include "miniz.c"
#else
#  include <zlib.h>
#endif
#include "zip.h"

/*
** Type of archive to build.
*/
#define ARCHIVE_ZIP   0
#define ARCHIVE_SQLAR 1

/*
** Write a 16- or 32-bit integer as little-endian into the given buffer.
*/
static void put16(char *z, int v){
  z[0] = v & 0xff;
  z[1] = (v>>8) & 0xff;
}

static int nEntry;   /* Number of files */
static int dosTime;  /* DOS-format time */
static int dosDate;  /* DOS-format date */
static int unixTime; /* Seconds since 1970 */
static int nDir;     /* Number of entries in azDir[] */
static char **azDir; /* Directory names already added to the archive */

typedef struct Archive Archive;
struct Archive {
  int eType;                      /* Type of archive (SQLAR or ZIP) */
  Blob *pBlob;                    /* Output blob */
  Blob tmp;                       /* Blob used as temp space for compression */
  sqlite3 *db;                    /* Db used to assemble sqlar archive */
  sqlite3_stmt *pInsert;          /* INSERT statement for SQLAR */
  sqlite3_vfs vfs;                /* VFS object */
};

/*
** Ensure that blob pBlob is at least nMin bytes in size.
*/
static void zip_blob_minsize(Blob *pBlob, int nMin){
  if( blob_size(pBlob)<nMin ){
    blob_resize(pBlob, nMin);
  }
}

/*************************************************************************
** Implementation of "archive" VFS. A VFS designed to store the contents
** of a new database in a Blob. Used to construct sqlar archives in
** memory.
*/
typedef struct ArchiveFile ArchiveFile;
struct ArchiveFile {
  sqlite3_file base;              /* Base class */
  Blob *pBlob;
};

static int archiveClose(sqlite3_file *pFile){
  return SQLITE_OK;
}
static int archiveRead(
    sqlite3_file *pFile, void *pBuf, int iAmt, sqlite3_int64 iOfst
){
  assert( iOfst==0 || iOfst==24 );
  return SQLITE_IOERR_SHORT_READ;
}
static int archiveWrite(
    sqlite3_file *pFile, const void *pBuf, int iAmt, sqlite3_int64 iOfst
){
  ArchiveFile *pAF = (ArchiveFile*)pFile;
  int nMin = (int)iOfst + iAmt;
  char *aBlob;                    /* Output buffer */

  zip_blob_minsize(pAF->pBlob, nMin);
  aBlob = blob_buffer(pAF->pBlob);
  memcpy(&aBlob[iOfst], pBuf, iAmt);
  return SQLITE_OK;
}
static int archiveTruncate(sqlite3_file *pFile, sqlite3_int64 size){
  return SQLITE_OK;
}
static int archiveSync(sqlite3_file *pFile, int flags){
  return SQLITE_OK;
}
static int archiveFileSize(sqlite3_file *pFile, sqlite3_int64 *pSize){
  *pSize = 0;
  return SQLITE_OK;
}
static int archiveLock(sqlite3_file *pFile, int eLock){
  return SQLITE_OK;
}
static int archiveUnlock(sqlite3_file *pFile, int eLock){
  return SQLITE_OK;
}
static int archiveCheckReservedLock(sqlite3_file *pFile, int *pResOut){
  *pResOut = 0;
  return SQLITE_OK;
}
static int archiveFileControl(sqlite3_file *pFile, int op, void *pArg){
  if( op==SQLITE_FCNTL_SIZE_HINT ){
    ArchiveFile *pAF = (ArchiveFile*)pFile;
    zip_blob_minsize(pAF->pBlob, (int)(*(sqlite3_int64*)pArg));
  }
  return SQLITE_NOTFOUND;
}
static int archiveSectorSize(sqlite3_file *pFile){
  return 512;
}
static int archiveDeviceCharacteristics(sqlite3_file *pFile){
  return 0;
}

static int archiveOpen(
  sqlite3_vfs *pVfs, const char *zName, 
  sqlite3_file *pFile, int flags, int *pOutFlags
){
  static struct sqlite3_io_methods methods = {
    1,             /* iVersion */
    archiveClose,
    archiveRead,
    archiveWrite,
    archiveTruncate,
    archiveSync,
    archiveFileSize,
    archiveLock,
    archiveUnlock,
    archiveCheckReservedLock,
    archiveFileControl,
    archiveSectorSize,
    archiveDeviceCharacteristics,
    0, 0, 0, 0,
    0, 0
  };

  ArchiveFile *pAF = (ArchiveFile*)pFile;
  assert( flags & SQLITE_OPEN_MAIN_DB );

  pAF->base.pMethods = &methods;
  pAF->pBlob = (Blob*)pVfs->pAppData;

  return SQLITE_OK;
}
static int archiveDelete(sqlite3_vfs *pVfs, const char *zName, int syncDir){
  return SQLITE_OK;
}
static int archiveAccess(
    sqlite3_vfs *pVfs, const char *zName, int flags, int *pResOut
){
  *pResOut = 0;
  return SQLITE_OK;
}
static int archiveFullPathname(
  sqlite3_vfs *pVfs, const char *zIn, int nOut, char *zOut
){
  int n = strlen(zIn);
  memcpy(zOut, zIn, n+1);
  return SQLITE_OK;
}
static int archiveRandomness(sqlite3_vfs *pVfs, int nByte, char *zOut){
  memset(zOut, 0, nByte);
  return SQLITE_OK;
}
static int archiveSleep(sqlite3_vfs *pVfs, int microseconds){
  return SQLITE_OK;
}
static int archiveCurrentTime(sqlite3_vfs *pVfs, double *prOut){
  return SQLITE_OK;
}
static int archiveGetLastError(sqlite3_vfs *pVfs, int nBuf, char *aBuf){
  return SQLITE_OK;
}
/*
** End of "archive" VFS.
*************************************************************************/

/*
** Initialize a new ZIP archive.
*/
void zip_open(void){
  blob_zero(&body);
  blob_zero(&toc);
  nEntry = 0;

void zip_set_timedate(double rDate){
  char *zDate = db_text(0, "SELECT datetime(%.17g)", rDate);
  zip_set_timedate_from_str(zDate);
  fossil_free(zDate);
  unixTime = (rDate - 2440587.5)*86400.0;
}



























/*
** Append a single file to a growing ZIP archive.
**
** pFile is the file to be appended.  zName is the name
** that the file should be saved as.
*/
static void zip_add_file_to_zip(
  Archive *p,
  const char *zName, 
  const Blob *pFile, 
  int mPerm
){
  z_stream stream;
  int nameLen;
  int toOut = 0;
  int iStart;
  int iCRC = 0;
  int nByte = 0;
  int nByteCompr = 0;

  blob_append(&toc, zBuf, 46);
  blob_append(&toc, zName, nameLen);
  put16(&zExTime[2], 5);
  blob_append(&toc, zExTime, 9);
  nEntry++;
}

static void zip_add_file_to_sqlar(
  Archive *p,
  const char *zName, 
  const Blob *pFile, 
  int mPerm
){
  int nName = strlen(zName);

  if( p->db==0 ){
    assert( p->vfs.zName==0 );
    p->vfs.zName = (const char*)mprintf("archivevfs%p", (void*)p);
    p->vfs.iVersion = 1;
    p->vfs.szOsFile = sizeof(ArchiveFile);
    p->vfs.mxPathname = 512;
    p->vfs.pAppData = (void*)p->pBlob;
    p->vfs.xOpen = archiveOpen;
    p->vfs.xDelete = archiveDelete;
    p->vfs.xAccess = archiveAccess;
    p->vfs.xFullPathname = archiveFullPathname;
    p->vfs.xRandomness = archiveRandomness;
    p->vfs.xSleep = archiveSleep;
    p->vfs.xCurrentTime = archiveCurrentTime;
    p->vfs.xGetLastError = archiveGetLastError;
    sqlite3_vfs_register(&p->vfs, 0);
    sqlite3_open_v2("file:xyz.db", &p->db, 
        SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE, p->vfs.zName
    );
    assert( p->db );
    blob_zero(&p->tmp);
    sqlite3_exec(p->db, 
        "PRAGMA journal_mode = off;"
        "PRAGMA cache_spill = off;"
        "BEGIN;"
        "CREATE TABLE sqlar("
          "name TEXT PRIMARY KEY,  -- name of the file\n"
          "mode INT,               -- access permissions\n"
          "mtime INT,              -- last modification time\n"
          "sz INT,                 -- original file size\n"
          "data BLOB               -- compressed content\n"
        ");", 0, 0, 0
    );
    sqlite3_prepare(p->db, 
        "INSERT INTO sqlar VALUES(?, ?, ?, ?, ?)", -1, 
        &p->pInsert, 0
    );
    assert( p->pInsert );

    sqlite3_bind_int64(p->pInsert, 3, unixTime);
    blob_zero(p->pBlob);
  }

  if( pFile==0 ){
    /* Directory. */
    if( zName[nName-1]=='/' ) nName--;
    sqlite3_bind_text(p->pInsert, 1, zName, nName, SQLITE_STATIC);
    sqlite3_bind_int(p->pInsert, 2, 040755);
    sqlite3_bind_int(p->pInsert, 4, 0);
    sqlite3_bind_null(p->pInsert, 5);
  }else{
    sqlite3_bind_text(p->pInsert, 1, zName, nName, SQLITE_STATIC);
    if( mPerm==PERM_LNK ){
      sqlite3_bind_int(p->pInsert, 2, 0120755);
      sqlite3_bind_int(p->pInsert, 4, -1);
      sqlite3_bind_text(p->pInsert, 5, 
          blob_buffer(pFile), blob_size(pFile), SQLITE_STATIC
      );
    }else{
      int nIn = blob_size(pFile);
      unsigned long int nOut = nIn;
      sqlite3_bind_int(p->pInsert, 2, mPerm==PERM_EXE ? 0100755 : 0100644);
      sqlite3_bind_int(p->pInsert, 4, nIn);
      zip_blob_minsize(&p->tmp, nIn);
      compress(
          blob_buffer(&p->tmp), &nOut, (unsigned char*)blob_buffer(pFile), nIn
      );
      if( nOut>=nIn ){
        sqlite3_bind_blob(p->pInsert, 5, 
            blob_buffer(pFile), blob_size(pFile), SQLITE_STATIC
        );
      }else{
        sqlite3_bind_blob(p->pInsert, 5, 
            blob_buffer(&p->tmp), nOut, SQLITE_STATIC
        );
      }
    }
  }

  sqlite3_step(p->pInsert);
  sqlite3_reset(p->pInsert);
}

static void zip_add_file(
  Archive *p,
  const char *zName, 
  const Blob *pFile, 
  int mPerm
){
  if( p->eType==ARCHIVE_ZIP ){
    zip_add_file_to_zip(p, zName, pFile, mPerm);
  }else{
    zip_add_file_to_sqlar(p, zName, pFile, mPerm);
  }
}

/*
** If the given filename includes one or more directory entries, make
** sure the directories are already in the archive.  If they are not
** in the archive, add them.
*/
static void zip_add_folders(Archive *p, char *zName){
  int i, c;
  int j;
  for(i=0; zName[i]; i++){
    if( zName[i]=='/' ){
      c = zName[i+1];
      zName[i+1] = 0;
      for(j=0; j<nDir; j++){
        if( fossil_strcmp(zName, azDir[j])==0 ) break;
      }
      if( j>=nDir ){
        nDir++;
        azDir = fossil_realloc(azDir, sizeof(azDir[0])*nDir);
        azDir[j] = mprintf("%s", zName);
        zip_add_file(p, zName, 0, 0);
      }
      zName[i+1] = c;
    }
  }
}

/*
** Free all the members of structure Archive allocated while processing
** an SQLAR request.
*/
static void free_archive(Archive *p){
  if( p->vfs.zName ){
    sqlite3_vfs_unregister(&p->vfs);
    fossil_free((char*)p->vfs.zName);
    p->vfs.zName = 0;
  }
  sqlite3_finalize(p->pInsert);
  p->pInsert = 0;
  sqlite3_close(p->db);
  p->db = 0;
}

/*
** Write the ZIP archive into the given BLOB.
*/
static void zip_close(Archive *p){


  int i;
  if( p->eType==ARCHIVE_ZIP ){
    int iTocStart;
    int iTocEnd;
    char zBuf[30];

    iTocStart = blob_size(&body);
    blob_append(&body, blob_buffer(&toc), blob_size(&toc));
    iTocEnd = blob_size(&body);

    memset(zBuf, 0, sizeof(zBuf));
    put32(&zBuf[0], 0x06054b50);
    put16(&zBuf[4], 0);
    put16(&zBuf[6], 0);
    put16(&zBuf[8], nEntry);
    put16(&zBuf[10], nEntry);
    put32(&zBuf[12], iTocEnd - iTocStart);
    put32(&zBuf[16], iTocStart);
    put16(&zBuf[20], 0);
    blob_append(&body, zBuf, 22);
    blob_reset(&toc);
    *(p->pBlob) = body;
    blob_zero(&body);
  }else{
    if( p->db ) sqlite3_exec(p->db, "COMMIT", 0, 0, 0);
    free_archive(p);
    blob_reset(&p->tmp);
  }

  nEntry = 0;
  for(i=0; i<nDir; i++){
    fossil_free(azDir[i]);
  }
  fossil_free(azDir);
  nDir = 0;
  azDir = 0;
}

/*
** COMMAND: test-filezip
**
** Generate a ZIP archive specified by the first argument that
** contains files given in the second and subsequent arguments.
*/
void filezip_cmd(void){
  int i;
  Blob zip;
  Blob file;
  int eFType = SymFILE;
  Archive sArchive;
  memset(&sArchive, 0, sizeof(Archive));
  sArchive.eType = ARCHIVE_ZIP;
  sArchive.pBlob = &zip;
  if( g.argc<3 ){
    usage("ARCHIVE FILE....");
  }
  if( find_option("dereference","h",0)!=0 ){
    eFType = ExtFILE;
  }
  zip_open();
  for(i=3; i<g.argc; i++){
    blob_zero(&file);
    blob_read_from_file(&file, g.argv[i], eFType);
    zip_add_file(&sArchive, g.argv[i], &file, file_perm(0,0));
    blob_reset(&file);
  }
  zip_close(&sArchive);
  blob_write_to_file(&zip, g.argv[2]);
}

/*
** Given the RID for a manifest, construct a ZIP archive containing
** all files in the corresponding baseline.
**

** zDir is a "synthetic" subdirectory which all zipped files get
** added to as part of the zip file. It may be 0 or an empty string,
** in which case it is ignored. The intention is to create a zip which
** politely expands into a subdir instead of filling your current dir
** with source files. For example, pass a UUID or "ProjectName".
**
*/
static void zip_of_checkin(
  int eType,          /* Type of archive (ZIP or SQLAR) */
  int rid,            /* The RID of the checkin to build the archive from */
  Blob *pZip,         /* Write the archive content into this blob */
  const char *zDir,   /* Top-level directory of the archive */
  Glob *pInclude,     /* Only include files that match this pattern */
  Glob *pExclude      /* Exclude files that match this pattern */
){
  Blob mfile, hash, file;
  Manifest *pManifest;
  ManifestFile *pFile;
  Blob filename;
  int nPrefix;

  Archive sArchive;
  memset(&sArchive, 0, sizeof(Archive));
  sArchive.eType = eType;
  sArchive.pBlob = pZip;

  content_get(rid, &mfile);
  if( blob_size(&mfile)==0 ){
    blob_zero(pZip);
    return;
  }
  blob_set_dynamic(&hash, rid_to_uuid(rid));
  blob_zero(&filename);

       && (flg & MFESTFLG_TAGS) ){
        eflg |= MFESTFLG_TAGS;
      }

      if( eflg & MFESTFLG_RAW ){
        blob_append(&filename, "manifest", -1);
        zName = blob_str(&filename);
        zip_add_folders(&sArchive, zName);
        sterilize_manifest(&mfile);
        zip_add_file(&sArchive, zName, &mfile, 0);
      }
      if( eflg & MFESTFLG_UUID ){
        blob_append(&hash, "\n", 1);
        blob_resize(&filename, nPrefix);
        blob_append(&filename, "manifest.uuid", -1);
        zName = blob_str(&filename);
        zip_add_folders(&sArchive, zName);
        zip_add_file(&sArchive, zName, &hash, 0);
      }
      if( eflg & MFESTFLG_TAGS ){
        Blob tagslist;
        blob_zero(&tagslist);
        get_checkin_taglist(rid, &tagslist);
        blob_resize(&filename, nPrefix);
        blob_append(&filename, "manifest.tags", -1);
        zName = blob_str(&filename);
        zip_add_folders(&sArchive, zName);
        zip_add_file(&sArchive, zName, &tagslist, 0);
        blob_reset(&tagslist);
      }
    }
    manifest_file_rewind(pManifest);
    while( (pFile = manifest_file_next(pManifest,0))!=0 ){
      int fid;
      if( pInclude!=0 && !glob_match(pInclude, pFile->zName) ) continue;
      if( glob_match(pExclude, pFile->zName) ) continue;
      fid = uuid_to_rid(pFile->zUuid, 0);
      if( fid ){
        content_get(fid, &file);
        blob_resize(&filename, nPrefix);
        blob_append(&filename, pFile->zName, -1);
        zName = blob_str(&filename);
        zip_add_folders(&sArchive, zName);
        zip_add_file(&sArchive, zName, &file, manifest_file_mperm(pFile));
        blob_reset(&file);
      }
    }
  }
  blob_reset(&mfile);
  manifest_destroy(pManifest);
  blob_reset(&filename);
  blob_reset(&hash);
  zip_close(&sArchive);
}

/*















** Implementation of zip_cmd and sqlar_cmd.




*/
static void archive_cmd(int eType){
  int rid;
  Blob zip;
  const char *zName;
  Glob *pInclude = 0;
  Glob *pExclude = 0;
  const char *zInclude;
  const char *zExclude;

  zName = find_option("name", 0, 1);
  zExclude = find_option("exclude", "X", 1);
  if( zExclude ) pExclude = glob_create(zExclude);
  zInclude = find_option("include", 0, 1);
  if( zInclude ) pInclude = glob_create(zInclude);
  db_find_and_open_repository(0, 0);

          " || substr(blob.uuid, 1, 10)"
       "  FROM event, blob"
       " WHERE event.objid=%d"
       "   AND blob.rid=%d",
       db_get("project-name", "unnamed"), rid, rid
    );
  }
  zip_of_checkin(eType, rid, &zip, zName, pInclude, pExclude);
  glob_free(pInclude);
  glob_free(pExclude);
  blob_write_to_file(&zip, g.argv[3]);
  blob_reset(&zip);
}

/*
** COMMAND: zip*
**
** Usage: %fossil zip VERSION OUTPUTFILE [OPTIONS]
**
** Generate a ZIP archive for a check-in.  If the --name option is
** used, its argument becomes the name of the top-level directory in the
** resulting ZIP archive.  If --name is omitted, the top-level directory
** name is derived from the project name, the check-in date and time, and
** the artifact ID of the check-in.
**
** The GLOBLIST argument to --exclude and --include can be a comma-separated
** list of glob patterns, where each glob pattern may optionally be enclosed
** in "..." or '...' so that it may contain commas.  If a file matches both
** --include and --exclude then it is excluded.
**
** Options:
**   -X|--exclude GLOBLIST   Comma-separated list of GLOBs of files to exclude
**   --include GLOBLIST      Comma-separated list of GLOBs of files to include
**   --name DIRECTORYNAME    The name of the top-level directory in the archive
**   -R REPOSITORY           Specify a Fossil repository
*/
void zip_cmd(void){
  archive_cmd(ARCHIVE_ZIP);
}

/*
** COMMAND: sqlar*
**
** Usage: %fossil sqlar VERSION OUTPUTFILE [OPTIONS]
**
** Generate an SQLAR archive for a check-in.  If the --name option is
** used, its argument becomes the name of the top-level directory in the
** resulting SQLAR archive.  If --name is omitted, the top-level directory
** name is derived from the project name, the check-in date and time, and
** the artifact ID of the check-in.
**
** The GLOBLIST argument to --exclude and --include can be a comma-separated
** list of glob patterns, where each glob pattern may optionally be enclosed
** in "..." or '...' so that it may contain commas.  If a file matches both
** --include and --exclude then it is excluded.
**
** Options:
**   -X|--exclude GLOBLIST   Comma-separated list of GLOBs of files to exclude
**   --include GLOBLIST      Comma-separated list of GLOBs of files to include
**   --name DIRECTORYNAME    The name of the top-level directory in the archive
**   -R REPOSITORY           Specify a Fossil repository
*/
void sqlar_cmd(void){
  archive_cmd(ARCHIVE_SQLAR);
}

/*
** WEBPAGE: zip
** URL: /zip
**
** Generate a ZIP archive for the check-in specified by the "r"
** query parameter.  Return that ZIP archive as the HTTP reply content.

    @ <input type="submit" value="Download" />
    @ </form>
    style_footer();
    return;
  }
  blob_zero(&zip);
  if( cache_read(&zip, zKey)==0 ){
    zip_of_checkin(ARCHIVE_ZIP, rid, &zip, zName, pInclude, pExclude);
    cache_write(&zip, zKey);
  }
  glob_free(pInclude);
  glob_free(pExclude);
  fossil_free(zName);
  fossil_free(zRid);
  blob_reset(&cacheKey);
  cgi_set_content(&zip);
  cgi_set_content_type("application/zip");
}