Fossil

Check-in [a4b59c32]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Use the fork() system call (when available) to start backoffice, in an attempt to avoid unseemly delays in upstream.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fork-backoffice
Files: files | file ages | folders
SHA3-256:a4b59c320796b31166851b6ea5e3d5505f23ce15bd55f9e87caa2e27e961d4ff
User & Date: drh 2018-08-07 15:50:26
Context
2018-08-07
17:48
Fix the separate-process backoffice so that it works smoothly on linux. Still work to be done on Windows. check-in: af7d67c6 user: drh tags: fork-backoffice
15:50
Use the fork() system call (when available) to start backoffice, in an attempt to avoid unseemly delays in upstream. check-in: a4b59c32 user: drh tags: fork-backoffice
15:12
A new implementation for "Forum" in which each forum post is an artifact. This merge includes lots of enhancements to email notification, backoffice, configuration, and other subsystems, all in support of the new forum artifacts. The forum feature is not complete nor bug-free but at this point it seems good enough to continue development on trunk. check-in: 99fcc43f user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/backoffice.c.

69
70
71
72
73
74
75


76
77
78
79
80
81
82
83
84




85
86
87
88


89
90
91
92
93
94
95
...
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219


220
221
222
223







224






225
226
227
228
229
230
231
232
233
234
235























236
237
238
239
240
241
242
...
301
302
303
304
305
306
307
308
309
310
311
312

313
314

315
316
317












































  sqlite3_uint64 idCurrent;   /* ID for the current lease holder */
  sqlite3_uint64 tmCurrent;   /* Expiration of the current lease */
  sqlite3_uint64 idNext;      /* ID for the next lease holder on queue */
  sqlite3_uint64 tmNext;      /* Expiration of the next lease */
};
#endif



/*
** Set to prevent backoffice processing from every entering sleep or
** otherwise taking a long time to complete.  Set this when a user-visible
** process might need to wait for backoffice to complete.
*/
static int backofficeNoDelay = 0;

/*
** Disable the backoffice




*/
void backoffice_no_delay(void){
  backofficeNoDelay = 1;
}



/*
** Parse a unsigned 64-bit integer from a string.  Return a pointer
** to the character of z[] that occurs after the integer.
*/
static const char *backofficeParseInt(const char *z, sqlite3_uint64 *pVal){
  *pVal = 0;
................................................................................
    sqlite3_uint64 x = (sqlite3_uint64)atoi(g.argv[i]);
    fossil_print("ProcessId %lld: exists %d done %d\n",
                 x, backofficeProcessExists(x),
                    backofficeProcessDone(x));
  }
}

/* This is the main public interface to the backoffice.  A process invokes this
** routine in an attempt to become the backoffice.  If another process is
** already working as the backoffice, this routine returns very quickly
** without doing any work - allowing the other process to continue.  But
** if no other processes are currently operating as the backoffice, this
** routine enters a loop to do background work periodically.
*/
void backoffice_run(void){
  Lease x;
  sqlite3_uint64 tmNow;


  sqlite3_uint64 idSelf;
  int lastWarning = 0;
  int warningDelay = 30;
  static int once = 0;














  if( once ){
    fossil_panic("multiple calls to backoffice_run()");
  }
  once = 1;
  if( g.db==0 ){
    fossil_panic("database not open for backoffice processing");
  }
  if( db_transaction_nesting_depth()!=0 ){
    fossil_panic("transaction %s not closed prior to backoffice processing",
                 db_transaction_start_point());
  }























  backofficeTimeout(BKOFCE_LEASE_TIME*2);
  idSelf = backofficeProcessId();
  while(1){
    tmNow = time(0);
    db_begin_write();
    backofficeReadLease(&x);
    if( x.tmNext>=tmNow
................................................................................
** backoffice processing tasks, add them here.
*/
void backoffice_work(void){
  email_backoffice(0);
}

/*
** COMMAND: test-backoffice
**
** Usage: test-backoffice
**
** Run backoffice processing

*/
void test_backoffice_command(void){

  db_find_and_open_repository(0,0);
  backoffice_run();
}



















































>
>
|
|





|
|
>
>
>
>

<
|
|
>
>







 







|
|
|
|
<
<

|


>
>
|
|
|
|
>
>
>
>
>
>
>
|
>
>
>
>
>
>
|
|

|







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|

|

|
>

|
>

|

>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91

92
93
94
95
96
97
98
99
100
101
102
...
210
211
212
213
214
215
216
217
218
219
220


221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
...
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
  sqlite3_uint64 idCurrent;   /* ID for the current lease holder */
  sqlite3_uint64 tmCurrent;   /* Expiration of the current lease */
  sqlite3_uint64 idNext;      /* ID for the next lease holder on queue */
  sqlite3_uint64 tmNext;      /* Expiration of the next lease */
};
#endif

/***************************************************************************
** Local state variables
**
** Set to prevent backoffice processing from ever entering sleep or
** otherwise taking a long time to complete.  Set this when a user-visible
** process might need to wait for backoffice to complete.
*/
static int backofficeNoDelay = 0;

/* This variable is set to the name of a database on which backoffice
** should run if backoffice process is needed.  It is set by the
** backoffice_check_if_needed() routine which must be run while the database
** file is open.  Later, after the database is closed, the
** backoffice_run_if_needed() will consult this variable to see if it
** should be a no-op.
*/

static char *backofficeDb = 0;

/* End of state variables
****************************************************************************/

/*
** Parse a unsigned 64-bit integer from a string.  Return a pointer
** to the character of z[] that occurs after the integer.
*/
static const char *backofficeParseInt(const char *z, sqlite3_uint64 *pVal){
  *pVal = 0;
................................................................................
    sqlite3_uint64 x = (sqlite3_uint64)atoi(g.argv[i]);
    fossil_print("ProcessId %lld: exists %d done %d\n",
                 x, backofficeProcessExists(x),
                    backofficeProcessDone(x));
  }
}

/*
** If backoffice processing is needed set the backofficeDb value to the
** name of the database file.  If no backoffice processing is needed,
** this routine makes no changes to state.


*/
void backoffice_check_if_needed(void){
  Lease x;
  sqlite3_uint64 tmNow;

  if( backofficeDb ) return;
  if( g.zRepositoryName==0 ) return;
  if( g.db==0 ) return;
  tmNow = time(0);
  backofficeReadLease(&x);
  if( x.tmNext>=tmNow && backofficeProcessExists(x.idNext) ){
    /* Another backoffice process is already queued up to run.  This
    ** process does not need to do any backoffice work. */
    return;
  }else{
    /* We need to run backup to be (at a minimum) on-deck */
    backofficeDb = fossil_strdup(g.zRepositoryName);
  }
}

/*
** Check for errors prior to running backoffice_thread() or backoffice_run().
*/
static void backoffice_error_check_one(int *pOnce){
  if( *pOnce ){
    fossil_panic("multiple calls to backoffice()");
  }
  *pOnce = 1;
  if( g.db==0 ){
    fossil_panic("database not open for backoffice processing");
  }
  if( db_transaction_nesting_depth()!=0 ){
    fossil_panic("transaction %s not closed prior to backoffice processing",
                 db_transaction_start_point());
  }
}

/* This is the main loop for backoffice processing.
**
** If others process is already working as the current backoffice and
** the on-deck backoffice, then this routine returns very quickly
** without doing any work.
**
** If no backoffice processes are running at all, this routine becomes
** the main backoffice.
**
** If a primary backoffice is running, but a on-deck backoffice is
** needed, this routine becomes that backoffice.
*/
static void backoffice_thread(void){
  Lease x;
  sqlite3_uint64 tmNow;
  sqlite3_uint64 idSelf;
  int lastWarning = 0;
  int warningDelay = 30;
  static int once = 0;

  backoffice_error_check_one(&once);
  backofficeTimeout(BKOFCE_LEASE_TIME*2);
  idSelf = backofficeProcessId();
  while(1){
    tmNow = time(0);
    db_begin_write();
    backofficeReadLease(&x);
    if( x.tmNext>=tmNow
................................................................................
** backoffice processing tasks, add them here.
*/
void backoffice_work(void){
  email_backoffice(0);
}

/*
** COMMAND: backoffice
**
** Usage: backoffice [-R repository]
**
** Run backoffice processing.  This might be done by a cron job or
** similar to make sure backoffice processing happens periodically.
*/
void backoffice_command(void){
  verify_all_options();
  db_find_and_open_repository(0,0);
  backoffice_thread();
}

/*
** This is the main interface to backoffice from the rest of the system.
** This routine launches either backoffice_thread() directly or as a
** subprocess.
*/
void backoffice_run_if_needed(void){
  if( backofficeDb==0 ) return;
  if( strcmp(backofficeDb,"x")==0 ) return;
  if( g.db ) return;
  if( g.repositoryOpen ) return;
#if !defined(_WIN32)
  {
    pid_t pid = fork();
    if( pid>0 ){
      /* This is the parent in a successful fork().  Return immediately. */
      if( g.fAnyTrace ){
        fprintf(stderr, "/***** Backoffice Child Creates as %d *****/\n",
                        (int)pid);
      }
      return;
    }
    if( pid==0 ){
      /* This is the child of a successful fork().  Run backoffice. */
      db_open_repository(backofficeDb);
      backofficeDb = "x";
      backoffice_thread();
      db_close(1);
      if( g.fAnyTrace ){
        fprintf(stderr, "/***** Backoffice Child %d exits *****/\n", getpid());
      }
      exit(0);
    }
  }
#endif
  /* Fork() failed or is unavailable.  Run backoffice in this process, but
  ** do so with the no-delay setting.
  */
  backofficeNoDelay = 1;
  db_open_repository(backofficeDb);
  backofficeDb = "x";
  backoffice_thread();
  db_close(1);
}

Changes to src/cgi.c.

352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
    g.httpOut = fossil_fopen("NUL", "wb");
#else
    g.httpOut = fossil_fopen("/dev/null", "wb");
#endif
    if( g.httpOut==0 ){
      fossil_warning("failed ot open /dev/null");
    }else{
      backoffice_run();
    }
  }
}

/*
** Do a redirect request to the URL given in the argument.
**







|







352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
    g.httpOut = fossil_fopen("NUL", "wb");
#else
    g.httpOut = fossil_fopen("/dev/null", "wb");
#endif
    if( g.httpOut==0 ){
      fossil_warning("failed ot open /dev/null");
    }else{
      backoffice_check_if_needed();
    }
  }
}

/*
** Do a redirect request to the URL given in the argument.
**

Changes to src/db.c.

1878
1879
1880
1881
1882
1883
1884

1885
1886
1887
1888
1889
1890
1891
    }
    g.db = 0;
  }
  g.repositoryOpen = 0;
  g.localOpen = 0;
  assert( g.dbConfig==0 );
  assert( g.zConfigDbName==0 );

}

/*
** Close the database as quickly as possible without unnecessary processing.
*/
void db_panic_close(void){
  if( g.db ){







>







1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
    }
    g.db = 0;
  }
  g.repositoryOpen = 0;
  g.localOpen = 0;
  assert( g.dbConfig==0 );
  assert( g.zConfigDbName==0 );
  backoffice_run_if_needed();
}

/*
** Close the database as quickly as possible without unnecessary processing.
*/
void db_panic_close(void){
  if( g.db ){

Changes to src/http_transport.c.

268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
** it is time to being receiving a reply.
*/
void transport_flip(UrlData *pUrlData){
  if( pUrlData->isFile ){
    char *zCmd;
    fclose(transport.pFile);
    zCmd = mprintf("\"%s\" http --in \"%s\" --out \"%s\" --ipaddr 127.0.0.1"
                   " \"%s\" --localauth --nodelay",
       g.nameOfExe, transport.zOutFile, transport.zInFile, pUrlData->name
    );
    fossil_system(zCmd);
    free(zCmd);
    transport.pFile = fossil_fopen(transport.zInFile, "rb");
  }
}







|







268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
** it is time to being receiving a reply.
*/
void transport_flip(UrlData *pUrlData){
  if( pUrlData->isFile ){
    char *zCmd;
    fclose(transport.pFile);
    zCmd = mprintf("\"%s\" http --in \"%s\" --out \"%s\" --ipaddr 127.0.0.1"
                   " \"%s\" --localauth",
       g.nameOfExe, transport.zOutFile, transport.zInFile, pUrlData->name
    );
    fossil_system(zCmd);
    free(zCmd);
    transport.pFile = fossil_fopen(transport.zInFile, "rb");
  }
}

Changes to src/main.c.

2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
....
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
**   --files GLOB     comma-separate glob patterns for static file to serve
**   --localauth      enable automatic login for local connections
**   --host NAME      specify hostname of the server
**   --https          signal a request coming in via https
**   --in FILE        Take input from FILE instead of standard input
**   --ipaddr ADDR    Assume the request comes from the given IP address
**   --nocompress     do not compress HTTP replies
**   --nodelay        omit backoffice processing if it would delay process exit
**   --nojail         drop root privilege but do not enter the chroot jail
**   --nossl          signal that no SSL connections are available
**   --notfound URL   use URL as "HTTP 404, object not found" page.
**   --out FILE       write results to FILE instead of to standard output
**   --repolist       If REPOSITORY is directory, URL "/" lists all repos
**   --scgi           Interpret input as SCGI rather than HTTP
**   --skin LABEL     Use override skin LABEL
................................................................................
    if( g.httpOut==0 ) fossil_fatal("cannot open \"%s\" for writing", zOutFile);
  }else{
    g.httpOut = stdout;
  }
  zIpAddr = find_option("ipaddr",0,1);
  useSCGI = find_option("scgi", 0, 0)!=0;
  zAltBase = find_option("baseurl", 0, 1);
  if( find_option("nodelay",0,0)!=0 ) backoffice_no_delay();
  if( zAltBase ) set_base_url(zAltBase);
  if( find_option("https",0,0)!=0 ){
    zIpAddr = fossil_getenv("REMOTE_HOST"); /* From stunnel */
    cgi_replace_parameter("HTTPS","on");
  }
  zHost = find_option("host", 0, 1);
  if( zHost ) cgi_replace_parameter("HTTP_HOST",zHost);







<







 







<







2295
2296
2297
2298
2299
2300
2301

2302
2303
2304
2305
2306
2307
2308
....
2361
2362
2363
2364
2365
2366
2367

2368
2369
2370
2371
2372
2373
2374
**   --files GLOB     comma-separate glob patterns for static file to serve
**   --localauth      enable automatic login for local connections
**   --host NAME      specify hostname of the server
**   --https          signal a request coming in via https
**   --in FILE        Take input from FILE instead of standard input
**   --ipaddr ADDR    Assume the request comes from the given IP address
**   --nocompress     do not compress HTTP replies

**   --nojail         drop root privilege but do not enter the chroot jail
**   --nossl          signal that no SSL connections are available
**   --notfound URL   use URL as "HTTP 404, object not found" page.
**   --out FILE       write results to FILE instead of to standard output
**   --repolist       If REPOSITORY is directory, URL "/" lists all repos
**   --scgi           Interpret input as SCGI rather than HTTP
**   --skin LABEL     Use override skin LABEL
................................................................................
    if( g.httpOut==0 ) fossil_fatal("cannot open \"%s\" for writing", zOutFile);
  }else{
    g.httpOut = stdout;
  }
  zIpAddr = find_option("ipaddr",0,1);
  useSCGI = find_option("scgi", 0, 0)!=0;
  zAltBase = find_option("baseurl", 0, 1);

  if( zAltBase ) set_base_url(zAltBase);
  if( find_option("https",0,0)!=0 ){
    zIpAddr = fossil_getenv("REMOTE_HOST"); /* From stunnel */
    cgi_replace_parameter("HTTPS","on");
  }
  zHost = find_option("host", 0, 1);
  if( zHost ) cgi_replace_parameter("HTTP_HOST",zHost);

Changes to src/winhttp.c.

403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
...
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
  }
  fossil_free(zIp);
  aux = fossil_fopen(zCmdFName, "wb");
  if( aux==0 ) goto end_request;
  fwrite(zCmd, 1, strlen(zCmd), aux);

  sqlite3_snprintf(sizeof(zCmd), zCmd,
    "\"%s\" http -args \"%s\" --nossl --nodelay%s",
    g.nameOfExe, zCmdFName, p->zOptions
  );
  in = fossil_fopen(zReplyFName, "w+b");
  fflush(out);
  fflush(aux);
  fossil_system(zCmd);
  if( in ){
................................................................................
    fwrite(zHdr, 1, got, out);
    wanted += got;
  }
  assert( g.zRepositoryName && g.zRepositoryName[0] );
  zIp = SocketAddr_toString(&p->addr);
  sqlite3_snprintf(sizeof(zCmd), zCmd,
    "\"%s\" http --in \"%s\" --out \"%s\" --ipaddr %s \"%s\""
    " --scgi --nossl --nodelay%s",
    g.nameOfExe, zRequestFName, zReplyFName, zIp,
    g.zRepositoryName, p->zOptions
  );
  fossil_free(zIp);
  in = fossil_fopen(zReplyFName, "w+b");
  fflush(out);
  fossil_system(zCmd);







|







 







|







403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
...
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
  }
  fossil_free(zIp);
  aux = fossil_fopen(zCmdFName, "wb");
  if( aux==0 ) goto end_request;
  fwrite(zCmd, 1, strlen(zCmd), aux);

  sqlite3_snprintf(sizeof(zCmd), zCmd,
    "\"%s\" http -args \"%s\" --nossl%s",
    g.nameOfExe, zCmdFName, p->zOptions
  );
  in = fossil_fopen(zReplyFName, "w+b");
  fflush(out);
  fflush(aux);
  fossil_system(zCmd);
  if( in ){
................................................................................
    fwrite(zHdr, 1, got, out);
    wanted += got;
  }
  assert( g.zRepositoryName && g.zRepositoryName[0] );
  zIp = SocketAddr_toString(&p->addr);
  sqlite3_snprintf(sizeof(zCmd), zCmd,
    "\"%s\" http --in \"%s\" --out \"%s\" --ipaddr %s \"%s\""
    " --scgi --nossl%s",
    g.nameOfExe, zRequestFName, zReplyFName, zIp,
    g.zRepositoryName, p->zOptions
  );
  fossil_free(zIp);
  in = fossil_fopen(zReplyFName, "w+b");
  fflush(out);
  fossil_system(zCmd);