Fossil

Check-in [c4f9b177]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Do the export within a transaction for performance. Record a complete two-way map of Git and Fossil names in the mirror.mmark table.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | mirror-cmd
Files: files | file ages | folders
SHA3-256: c4f9b177f4dfa1891ac1aaa817abae821928e3c1e90a997955ca579aac9576d4
User & Date: drh 2019-03-15 21:36:23.936
Context
2019-03-16
00:26
Enhance the "fossil git export" command so that it also exports tags as lightweight Git tags. ... (Closed-Leaf check-in: 4f19d67b user: drh tags: mirror-cmd)
2019-03-15
21:36
Do the export within a transaction for performance. Record a complete two-way map of Git and Fossil names in the mirror.mmark table. ... (check-in: c4f9b177 user: drh tags: mirror-cmd)
20:01
Add the "FossilOrigin-Name:" footer on all exported comments. ... (check-in: 051cd382 user: drh tags: mirror-cmd)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/export.c.
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
    }
  }
  zOut[j++] = '"';
  zOut[j] = 0;
  return zOut;
}

/*
** Transfer a tag over to the mirror.  "rid" is the BLOB.RID value for
** the record that describes the tag.
**
** The Git tag mechanism is very limited compared to Fossil.  Many Fossil
** tags cannot be exported to Git.  If this tag cannot be exported, then
** silently ignore it.
*/
static void gitmirror_send_tag(FILE *xCmd, int rid){
  return;
}

/*
** Locate the mark for a UUID.
**
** If the mark does not exist and if the bCreate flag is false, then
** return 0.  If the mark does not exist and the bCreate flag is true,
** then create the mark.
*/







<
<
<
<
<
<
<
<
<
<
<
<







910
911
912
913
914
915
916












917
918
919
920
921
922
923
    }
  }
  zOut[j++] = '"';
  zOut[j] = 0;
  return zOut;
}













/*
** Locate the mark for a UUID.
**
** If the mark does not exist and if the bCreate flag is false, then
** return 0.  If the mark does not exist and the bCreate flag is true,
** then create the mark.
*/
1208
1209
1210
1211
1212
1213
1214

1215
1216
1217
1218
1219
1220
1221
1222

1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
  z = mprintf("%s/.mirror_state", zMirror);
  rc = file_mkdir(z, ExtFILE, 0);
  if( rc ) fossil_fatal("cannot create directory \"%s\"", z);
  fossil_free(z);

  /* Attach the .mirror_state/db database */
  db_multi_exec("ATTACH '%q/.mirror_state/db' AS mirror;", zMirror);

  db_multi_exec(
    "CREATE TABLE IF NOT EXISTS mirror.mconfig(\n"
    "  key TEXT PRIMARY KEY,\n"
    "  Value ANY\n"
    ") WITHOUT ROWID;\n"
    "CREATE TABLE IF NOT EXISTS mirror.mmark(\n"
    "  id INTEGER PRIMARY KEY,\n"
    "  uuid TEXT UNIQUE\n"

    ");"
    "CREATE TABLE IF NOT EXISTS mirror.mtag(\n"
    "  tagname TEXT PRIMARY KEY,\n"
    "  cnt INTEGER DEFAULT 1\n"
    ") WITHOUT ROWID;"
  );

  /* See if there is any work to be done.  Exit early if not, before starting
  ** the "git fast-import" command. */
  if( !db_exists("SELECT 1 FROM event WHERE type IN ('t','ci')"
                 " AND mtime>coalesce((SELECT value FROM mconfig"
                                        " WHERE key='start'),0.0)")
  ){
    fossil_print("no changes\n");
    return;
  }








>







|
>

<
<
<
<




|







1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213




1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
  z = mprintf("%s/.mirror_state", zMirror);
  rc = file_mkdir(z, ExtFILE, 0);
  if( rc ) fossil_fatal("cannot create directory \"%s\"", z);
  fossil_free(z);

  /* Attach the .mirror_state/db database */
  db_multi_exec("ATTACH '%q/.mirror_state/db' AS mirror;", zMirror);
  db_begin_write();
  db_multi_exec(
    "CREATE TABLE IF NOT EXISTS mirror.mconfig(\n"
    "  key TEXT PRIMARY KEY,\n"
    "  Value ANY\n"
    ") WITHOUT ROWID;\n"
    "CREATE TABLE IF NOT EXISTS mirror.mmark(\n"
    "  id INTEGER PRIMARY KEY,\n"
    "  uuid TEXT UNIQUE,\n"
    "  githash TEXT UNIQUE\n"
    ");"




  );

  /* See if there is any work to be done.  Exit early if not, before starting
  ** the "git fast-import" command. */
  if( !db_exists("SELECT 1 FROM event WHERE type='ci'"
                 " AND mtime>coalesce((SELECT value FROM mconfig"
                                        " WHERE key='start'),0.0)")
  ){
    fossil_print("no changes\n");
    return;
  }

1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292

1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
    }
    fossil_free(zCmd);
  }

  /* Run the export */
  rEnd = 0.0;
  db_multi_exec(
    "CREATE TEMP TABLE tomirror(objid INTEGER PRIMARY KEY,type,mtime,uuid);\n"
    "INSERT INTO tomirror "
    "SELECT objid, type, mtime, blob.uuid FROM event, blob\n"
    " WHERE type IN ('ci','t')"
    "   AND mtime>coalesce((SELECT value FROM mconfig WHERE key='start'),0.0)"
    "   AND blob.rid=event.objid"
    "   AND blob.uuid NOT IN (SELECT uuid FROM mirror.mmark);"
  );
  nTotal = db_int(0, "SELECT count(*) FROM tomirror WHERE type='ci'");
  if( nLimit<nTotal ){
    nTotal = nLimit;
  }else if( nLimit>nTotal ){
    nLimit = nTotal;
  }
  db_prepare(&q,
    "SELECT objid, type, mtime, uuid FROM tomirror ORDER BY mtime"
  );
  while( nLimit && db_step(&q)==SQLITE_ROW ){
    double rMTime = db_column_double(&q, 2);
    const char *zType = db_column_text(&q, 1);
    int rid = db_column_int(&q, 0);

    const char *zUuid = db_column_text(&q, 3);
    if( rMTime>rEnd ) rEnd = rMTime;
    if( zType[0]=='t' ){
      gitmirror_send_tag(xCmd, rid);
    }else{
      gitmirror_send_checkin(xCmd, rid, zUuid, &nLimit, fManifest);
      printf("\r%d/%d  ", nTotal-nLimit, nTotal);
      fflush(stdout);
    }
  }
  db_finalize(&q);
  db_prepare(&q, "REPLACE INTO mirror.mconfig(key,value) VALUES('start',:x)");
  db_bind_double(&q, ":x", rEnd);
  db_step(&q);
  db_finalize(&q);
  fprintf(xCmd, "done\n");







|

|
|




|






|


<
<

>
|

<
<
<
|
|
|
<







1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275


1276
1277
1278
1279



1280
1281
1282

1283
1284
1285
1286
1287
1288
1289
    }
    fossil_free(zCmd);
  }

  /* Run the export */
  rEnd = 0.0;
  db_multi_exec(
    "CREATE TEMP TABLE tomirror(objid,mtime,uuid);\n"
    "INSERT INTO tomirror "
    "SELECT objid, mtime, blob.uuid FROM event, blob\n"
    " WHERE type='ci'"
    "   AND mtime>coalesce((SELECT value FROM mconfig WHERE key='start'),0.0)"
    "   AND blob.rid=event.objid"
    "   AND blob.uuid NOT IN (SELECT uuid FROM mirror.mmark);"
  );
  nTotal = db_int(0, "SELECT count(*) FROM tomirror");
  if( nLimit<nTotal ){
    nTotal = nLimit;
  }else if( nLimit>nTotal ){
    nLimit = nTotal;
  }
  db_prepare(&q,
    "SELECT objid, mtime, uuid FROM tomirror ORDER BY mtime"
  );
  while( nLimit && db_step(&q)==SQLITE_ROW ){


    int rid = db_column_int(&q, 0);
    double rMTime = db_column_double(&q, 1);
    const char *zUuid = db_column_text(&q, 2);
    if( rMTime>rEnd ) rEnd = rMTime;



    gitmirror_send_checkin(xCmd, rid, zUuid, &nLimit, fManifest);
    printf("\r%d/%d  ", nTotal-nLimit, nTotal);
    fflush(stdout);

  }
  db_finalize(&q);
  db_prepare(&q, "REPLACE INTO mirror.mconfig(key,value) VALUES('start',:x)");
  db_bind_double(&q, ":x", rEnd);
  db_step(&q);
  db_finalize(&q);
  fprintf(xCmd, "done\n");
1318
1319
1320
1321
1322
1323
1324

1325












1326
1327

1328
1329
1330
1331
1332
1333

1334
1335
1336
1337
1338
1339
1340
  */
  pOut = fopen(".mirror_state/out", "rb");
  if( pOut ){
    pIn = fopen(".mirror_state/in", "ab");
    if( pIn==0 ){
      fossil_fatal("cannot open %s/.mirror_state/in for appending", zMirror);
    }

    while( fgets(zLine, sizeof(zLine), pOut) ){












      fputs(zLine, pIn);
    }

    fclose(pOut);
    fclose(pIn);
    file_delete(".mirror_state/out");
  }else{
    fossil_fatal("git fast-import didn't generate a marks file!");
  }


  /* Optionally do a "git push" */
}

/*
** COMMAND: git
**







>

>
>
>
>
>
>
>
>
>
>
>
>


>






>







1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
  */
  pOut = fopen(".mirror_state/out", "rb");
  if( pOut ){
    pIn = fopen(".mirror_state/in", "ab");
    if( pIn==0 ){
      fossil_fatal("cannot open %s/.mirror_state/in for appending", zMirror);
    }
    db_prepare(&q, "UPDATE mirror.mmark SET githash=:githash WHERE id=:id");
    while( fgets(zLine, sizeof(zLine), pOut) ){
      int j, k;
      if( zLine[0]!=':' ) continue;
      db_bind_int(&q, ":id", atoi(zLine+1));
      for(j=1; zLine[j] && zLine[j]!=' '; j++){}
      if( zLine[j]!=' ' ) continue;
      j++;
      if( zLine[j]==0 ) continue;
      for(k=j; fossil_isalnum(zLine[k]); k++){}
      zLine[k] = 0;
      db_bind_text(&q, ":githash", &zLine[j]);
      db_step(&q);
      db_reset(&q);
      fputs(zLine, pIn);
    }
    db_finalize(&q);
    fclose(pOut);
    fclose(pIn);
    file_delete(".mirror_state/out");
  }else{
    fossil_fatal("git fast-import didn't generate a marks file!");
  }
  db_commit_transaction();

  /* Optionally do a "git push" */
}

/*
** COMMAND: git
**