Fossil

Check-in [a4b59c32]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Use the fork() system call (when available) to start backoffice, in an attempt to avoid unseemly delays in upstream.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fork-backoffice
Files: files | file ages | folders
SHA3-256:a4b59c320796b31166851b6ea5e3d5505f23ce15bd55f9e87caa2e27e961d4ff
User & Date: drh 2018-08-07 15:50:26
Context
2018-08-07
17:48
Fix the separate-process backoffice so that it works smoothly on linux. Still work to be done on Windows. check-in: af7d67c6 user: drh tags: fork-backoffice
15:50
Use the fork() system call (when available) to start backoffice, in an attempt to avoid unseemly delays in upstream. check-in: a4b59c32 user: drh tags: fork-backoffice
15:12
A new implementation for "Forum" in which each forum post is an artifact. This merge includes lots of enhancements to email notification, backoffice, configuration, and other subsystems, all in support of the new forum artifacts. The forum feature is not complete nor bug-free but at this point it seems good enough to continue development on trunk. check-in: 99fcc43f user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/backoffice.c.

    69     69     sqlite3_uint64 idCurrent;   /* ID for the current lease holder */
    70     70     sqlite3_uint64 tmCurrent;   /* Expiration of the current lease */
    71     71     sqlite3_uint64 idNext;      /* ID for the next lease holder on queue */
    72     72     sqlite3_uint64 tmNext;      /* Expiration of the next lease */
    73     73   };
    74     74   #endif
    75     75   
    76         -/*
    77         -** Set to prevent backoffice processing from every entering sleep or
           76  +/***************************************************************************
           77  +** Local state variables
           78  +**
           79  +** Set to prevent backoffice processing from ever entering sleep or
    78     80   ** otherwise taking a long time to complete.  Set this when a user-visible
    79     81   ** process might need to wait for backoffice to complete.
    80     82   */
    81     83   static int backofficeNoDelay = 0;
    82     84   
    83         -/*
    84         -** Disable the backoffice
           85  +/* This variable is set to the name of a database on which backoffice
           86  +** should run if backoffice process is needed.  It is set by the
           87  +** backoffice_check_if_needed() routine which must be run while the database
           88  +** file is open.  Later, after the database is closed, the
           89  +** backoffice_run_if_needed() will consult this variable to see if it
           90  +** should be a no-op.
    85     91   */
    86         -void backoffice_no_delay(void){
    87         -  backofficeNoDelay = 1;
    88         -}
           92  +static char *backofficeDb = 0;
           93  +
           94  +/* End of state variables
           95  +****************************************************************************/
    89     96   
    90     97   /*
    91     98   ** Parse a unsigned 64-bit integer from a string.  Return a pointer
    92     99   ** to the character of z[] that occurs after the integer.
    93    100   */
    94    101   static const char *backofficeParseInt(const char *z, sqlite3_uint64 *pVal){
    95    102     *pVal = 0;
................................................................................
   203    210       sqlite3_uint64 x = (sqlite3_uint64)atoi(g.argv[i]);
   204    211       fossil_print("ProcessId %lld: exists %d done %d\n",
   205    212                    x, backofficeProcessExists(x),
   206    213                       backofficeProcessDone(x));
   207    214     }
   208    215   }
   209    216   
   210         -/* This is the main public interface to the backoffice.  A process invokes this
   211         -** routine in an attempt to become the backoffice.  If another process is
   212         -** already working as the backoffice, this routine returns very quickly
   213         -** without doing any work - allowing the other process to continue.  But
   214         -** if no other processes are currently operating as the backoffice, this
   215         -** routine enters a loop to do background work periodically.
          217  +/*
          218  +** If backoffice processing is needed set the backofficeDb value to the
          219  +** name of the database file.  If no backoffice processing is needed,
          220  +** this routine makes no changes to state.
   216    221   */
   217         -void backoffice_run(void){
          222  +void backoffice_check_if_needed(void){
   218    223     Lease x;
   219    224     sqlite3_uint64 tmNow;
   220         -  sqlite3_uint64 idSelf;
   221         -  int lastWarning = 0;
   222         -  int warningDelay = 30;
   223         -  static int once = 0;
          225  +
          226  +  if( backofficeDb ) return;
          227  +  if( g.zRepositoryName==0 ) return;
          228  +  if( g.db==0 ) return;
          229  +  tmNow = time(0);
          230  +  backofficeReadLease(&x);
          231  +  if( x.tmNext>=tmNow && backofficeProcessExists(x.idNext) ){
          232  +    /* Another backoffice process is already queued up to run.  This
          233  +    ** process does not need to do any backoffice work. */
          234  +    return;
          235  +  }else{
          236  +    /* We need to run backup to be (at a minimum) on-deck */
          237  +    backofficeDb = fossil_strdup(g.zRepositoryName);
          238  +  }
          239  +}
   224    240   
   225         -  if( once ){
   226         -    fossil_panic("multiple calls to backoffice_run()");
          241  +/*
          242  +** Check for errors prior to running backoffice_thread() or backoffice_run().
          243  +*/
          244  +static void backoffice_error_check_one(int *pOnce){
          245  +  if( *pOnce ){
          246  +    fossil_panic("multiple calls to backoffice()");
   227    247     }
   228         -  once = 1;
          248  +  *pOnce = 1;
   229    249     if( g.db==0 ){
   230    250       fossil_panic("database not open for backoffice processing");
   231    251     }
   232    252     if( db_transaction_nesting_depth()!=0 ){
   233    253       fossil_panic("transaction %s not closed prior to backoffice processing",
   234    254                    db_transaction_start_point());
   235    255     }
          256  +}
          257  +
          258  +/* This is the main loop for backoffice processing.
          259  +**
          260  +** If others process is already working as the current backoffice and
          261  +** the on-deck backoffice, then this routine returns very quickly
          262  +** without doing any work.
          263  +**
          264  +** If no backoffice processes are running at all, this routine becomes
          265  +** the main backoffice.
          266  +**
          267  +** If a primary backoffice is running, but a on-deck backoffice is
          268  +** needed, this routine becomes that backoffice.
          269  +*/
          270  +static void backoffice_thread(void){
          271  +  Lease x;
          272  +  sqlite3_uint64 tmNow;
          273  +  sqlite3_uint64 idSelf;
          274  +  int lastWarning = 0;
          275  +  int warningDelay = 30;
          276  +  static int once = 0;
          277  +
          278  +  backoffice_error_check_one(&once);
   236    279     backofficeTimeout(BKOFCE_LEASE_TIME*2);
   237    280     idSelf = backofficeProcessId();
   238    281     while(1){
   239    282       tmNow = time(0);
   240    283       db_begin_write();
   241    284       backofficeReadLease(&x);
   242    285       if( x.tmNext>=tmNow
................................................................................
   301    344   ** backoffice processing tasks, add them here.
   302    345   */
   303    346   void backoffice_work(void){
   304    347     email_backoffice(0);
   305    348   }
   306    349   
   307    350   /*
   308         -** COMMAND: test-backoffice
          351  +** COMMAND: backoffice
   309    352   **
   310         -** Usage: test-backoffice
          353  +** Usage: backoffice [-R repository]
   311    354   **
   312         -** Run backoffice processing
          355  +** Run backoffice processing.  This might be done by a cron job or
          356  +** similar to make sure backoffice processing happens periodically.
   313    357   */
   314         -void test_backoffice_command(void){
          358  +void backoffice_command(void){
          359  +  verify_all_options();
   315    360     db_find_and_open_repository(0,0);
   316         -  backoffice_run();
          361  +  backoffice_thread();
          362  +}
          363  +
          364  +/*
          365  +** This is the main interface to backoffice from the rest of the system.
          366  +** This routine launches either backoffice_thread() directly or as a
          367  +** subprocess.
          368  +*/
          369  +void backoffice_run_if_needed(void){
          370  +  if( backofficeDb==0 ) return;
          371  +  if( strcmp(backofficeDb,"x")==0 ) return;
          372  +  if( g.db ) return;
          373  +  if( g.repositoryOpen ) return;
          374  +#if !defined(_WIN32)
          375  +  {
          376  +    pid_t pid = fork();
          377  +    if( pid>0 ){
          378  +      /* This is the parent in a successful fork().  Return immediately. */
          379  +      if( g.fAnyTrace ){
          380  +        fprintf(stderr, "/***** Backoffice Child Creates as %d *****/\n",
          381  +                        (int)pid);
          382  +      }
          383  +      return;
          384  +    }
          385  +    if( pid==0 ){
          386  +      /* This is the child of a successful fork().  Run backoffice. */
          387  +      db_open_repository(backofficeDb);
          388  +      backofficeDb = "x";
          389  +      backoffice_thread();
          390  +      db_close(1);
          391  +      if( g.fAnyTrace ){
          392  +        fprintf(stderr, "/***** Backoffice Child %d exits *****/\n", getpid());
          393  +      }
          394  +      exit(0);
          395  +    }
          396  +  }
          397  +#endif
          398  +  /* Fork() failed or is unavailable.  Run backoffice in this process, but
          399  +  ** do so with the no-delay setting.
          400  +  */
          401  +  backofficeNoDelay = 1;
          402  +  db_open_repository(backofficeDb);
          403  +  backofficeDb = "x";
          404  +  backoffice_thread();
          405  +  db_close(1);
   317    406   }

Changes to src/cgi.c.

   352    352       g.httpOut = fossil_fopen("NUL", "wb");
   353    353   #else
   354    354       g.httpOut = fossil_fopen("/dev/null", "wb");
   355    355   #endif
   356    356       if( g.httpOut==0 ){
   357    357         fossil_warning("failed ot open /dev/null");
   358    358       }else{
   359         -      backoffice_run();
          359  +      backoffice_check_if_needed();
   360    360       }
   361    361     }
   362    362   }
   363    363   
   364    364   /*
   365    365   ** Do a redirect request to the URL given in the argument.
   366    366   **

Changes to src/db.c.

  1878   1878       }
  1879   1879       g.db = 0;
  1880   1880     }
  1881   1881     g.repositoryOpen = 0;
  1882   1882     g.localOpen = 0;
  1883   1883     assert( g.dbConfig==0 );
  1884   1884     assert( g.zConfigDbName==0 );
         1885  +  backoffice_run_if_needed();
  1885   1886   }
  1886   1887   
  1887   1888   /*
  1888   1889   ** Close the database as quickly as possible without unnecessary processing.
  1889   1890   */
  1890   1891   void db_panic_close(void){
  1891   1892     if( g.db ){

Changes to src/http_transport.c.

   268    268   ** it is time to being receiving a reply.
   269    269   */
   270    270   void transport_flip(UrlData *pUrlData){
   271    271     if( pUrlData->isFile ){
   272    272       char *zCmd;
   273    273       fclose(transport.pFile);
   274    274       zCmd = mprintf("\"%s\" http --in \"%s\" --out \"%s\" --ipaddr 127.0.0.1"
   275         -                   " \"%s\" --localauth --nodelay",
          275  +                   " \"%s\" --localauth",
   276    276          g.nameOfExe, transport.zOutFile, transport.zInFile, pUrlData->name
   277    277       );
   278    278       fossil_system(zCmd);
   279    279       free(zCmd);
   280    280       transport.pFile = fossil_fopen(transport.zInFile, "rb");
   281    281     }
   282    282   }

Changes to src/main.c.

  2295   2295   **   --files GLOB     comma-separate glob patterns for static file to serve
  2296   2296   **   --localauth      enable automatic login for local connections
  2297   2297   **   --host NAME      specify hostname of the server
  2298   2298   **   --https          signal a request coming in via https
  2299   2299   **   --in FILE        Take input from FILE instead of standard input
  2300   2300   **   --ipaddr ADDR    Assume the request comes from the given IP address
  2301   2301   **   --nocompress     do not compress HTTP replies
  2302         -**   --nodelay        omit backoffice processing if it would delay process exit
  2303   2302   **   --nojail         drop root privilege but do not enter the chroot jail
  2304   2303   **   --nossl          signal that no SSL connections are available
  2305   2304   **   --notfound URL   use URL as "HTTP 404, object not found" page.
  2306   2305   **   --out FILE       write results to FILE instead of to standard output
  2307   2306   **   --repolist       If REPOSITORY is directory, URL "/" lists all repos
  2308   2307   **   --scgi           Interpret input as SCGI rather than HTTP
  2309   2308   **   --skin LABEL     Use override skin LABEL
................................................................................
  2362   2361       if( g.httpOut==0 ) fossil_fatal("cannot open \"%s\" for writing", zOutFile);
  2363   2362     }else{
  2364   2363       g.httpOut = stdout;
  2365   2364     }
  2366   2365     zIpAddr = find_option("ipaddr",0,1);
  2367   2366     useSCGI = find_option("scgi", 0, 0)!=0;
  2368   2367     zAltBase = find_option("baseurl", 0, 1);
  2369         -  if( find_option("nodelay",0,0)!=0 ) backoffice_no_delay();
  2370   2368     if( zAltBase ) set_base_url(zAltBase);
  2371   2369     if( find_option("https",0,0)!=0 ){
  2372   2370       zIpAddr = fossil_getenv("REMOTE_HOST"); /* From stunnel */
  2373   2371       cgi_replace_parameter("HTTPS","on");
  2374   2372     }
  2375   2373     zHost = find_option("host", 0, 1);
  2376   2374     if( zHost ) cgi_replace_parameter("HTTP_HOST",zHost);

Changes to src/winhttp.c.

   403    403     }
   404    404     fossil_free(zIp);
   405    405     aux = fossil_fopen(zCmdFName, "wb");
   406    406     if( aux==0 ) goto end_request;
   407    407     fwrite(zCmd, 1, strlen(zCmd), aux);
   408    408   
   409    409     sqlite3_snprintf(sizeof(zCmd), zCmd,
   410         -    "\"%s\" http -args \"%s\" --nossl --nodelay%s",
          410  +    "\"%s\" http -args \"%s\" --nossl%s",
   411    411       g.nameOfExe, zCmdFName, p->zOptions
   412    412     );
   413    413     in = fossil_fopen(zReplyFName, "w+b");
   414    414     fflush(out);
   415    415     fflush(aux);
   416    416     fossil_system(zCmd);
   417    417     if( in ){
................................................................................
   474    474       fwrite(zHdr, 1, got, out);
   475    475       wanted += got;
   476    476     }
   477    477     assert( g.zRepositoryName && g.zRepositoryName[0] );
   478    478     zIp = SocketAddr_toString(&p->addr);
   479    479     sqlite3_snprintf(sizeof(zCmd), zCmd,
   480    480       "\"%s\" http --in \"%s\" --out \"%s\" --ipaddr %s \"%s\""
   481         -    " --scgi --nossl --nodelay%s",
          481  +    " --scgi --nossl%s",
   482    482       g.nameOfExe, zRequestFName, zReplyFName, zIp,
   483    483       g.zRepositoryName, p->zOptions
   484    484     );
   485    485     fossil_free(zIp);
   486    486     in = fossil_fopen(zReplyFName, "w+b");
   487    487     fflush(out);
   488    488     fossil_system(zCmd);